{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import torch\n",
    "from transformers import *\n",
    "from transformers import AdamW\n",
    "import torch.utils.data as Data\n",
    "import collections\n",
    "import os\n",
    "import random\n",
    "import tarfile\n",
    "import torch\n",
    "from torch import nn\n",
    "import torchtext.vocab as Vocab\n",
    "import pickle as pk\n",
    "import copy\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import torch.nn.functional as F\n",
    "from IPython.display import display,HTML\n",
    "import os\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from torch.nn.utils.rnn import pack_padded_sequence\n",
    "from torch.nn.utils.rnn import pad_packed_sequence\n",
    "from torch.nn.utils.rnn import pack_sequence\n",
    "from torch.nn import CrossEntropyLoss, MSELoss\n",
    "from torchcrf import CRF\n",
    "from sklearn import metrics\n",
    "import joblib\n",
    "import math\n",
    "device=torch.device(\"cuda:2\" if torch.cuda.is_available() else \"cpu\")\n",
    "import argparse\n",
    "import glob\n",
    "import json\n",
    "import logging\n",
    "import unicodedata\n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset\n",
    "from torch.utils.data.distributed import DistributedSampler\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "import torch.utils.data as Data\n",
    "import jieba\n",
    "import jieba.posseg as pseg\n",
    "logger = logging.getLogger(__name__)\n",
    "logging.basicConfig(\n",
    "    format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n",
    "    datefmt=\"%m/%d/%Y %H:%M:%S\",\n",
    "    level=logging.INFO \n",
    ")\n",
    "ARG=collections.namedtuple('ARG',['train_batch_size',\n",
    " 'eval_batch_size',\n",
    " 'weight_decay',\n",
    " 'learning_rate',\n",
    " 'adam_epsilon',\n",
    " 'num_train_epochs',\n",
    " 'warmup_steps',\n",
    " 'gradient_accumulation_steps',\n",
    " 'save_steps',\n",
    " 'max_grad_norm',\n",
    " 'model_name_or_path',\n",
    " 'output_dir',\n",
    " 'seed',\n",
    " 'device',\n",
    " 'n_gpu',\n",
    " 'max_steps',\n",
    " 'output_mode',\n",
    "'fp16_opt_level',\n",
    "'fp16',\n",
    "'card_list'])\n",
    "parser = argparse.ArgumentParser()\n",
    "parser.add_argument(\"-c\",\"--card\", help=\"feature\",type=int,default=0)\n",
    "parser.add_argument(\"-rnn\",\"--rnn\", help=\"feature\",type=int,default=0)\n",
    "parser.add_argument(\"-cnn\",\"--cnn\", help=\"feature\",type=int,default=0)\n",
    "parser.add_argument(\"-plan\",\"--use_plan\", help=\"feature\",type=int,default=0)\n",
    "args = parser.parse_args()\n",
    "output_dir=\"./output_ner_pointer\"+str(args.rnn)+\"_\"+str(args.cnn)+\"/\"\n",
    "print(output_dir)\n",
    "print(\"cuda:\"+str(args.card))\n",
    "need_rnn=bool(args.rnn)\n",
    "need_cnn=bool(args.cnn)\n",
    "use_plan=bool(args.use_plan)\n",
    "print(\"need_rnn:\",need_rnn)\n",
    "print(\"need_cnn:\",need_cnn)\n",
    "print(\"use_plan:\",use_plan)\n",
    "print(\"cuda:\"+str(args.card))\n",
    "device=torch.device(\"cuda:\"+str(args.card) if torch.cuda.is_available() else \"cpu\")\n",
    "if args.card==0:\n",
    "    card_list=[0,1]\n",
    "if args.card==2:\n",
    "    card_list=[2,3]\n",
    "print(\"card_list\",card_list) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "code_folding": [
     0,
     16,
     35,
     53,
     63,
     86,
     101,
     121,
     147,
     151,
     156,
     162,
     164,
     188,
     237,
     350,
     390,
     457,
     504,
     530,
     559,
     589,
     622,
     721,
     821
    ]
   },
   "outputs": [],
   "source": [
    "def convert_text_to_ids(tokenizer, text, max_len=100):\n",
    "    if isinstance(text,str):\n",
    "        output=tokenizer.encode_plus(text,max_length=max_len,pad_to_max_length=True,return_tensors=\"pt\")\n",
    "        input_ids=output[\"input_ids\"].squeeze(0)\n",
    "        token_type_ids=output[\"token_type_ids\"].squeeze(0)\n",
    "        attention_mask=output[\"attention_mask\"].squeeze(0)\n",
    "    elif isinstance(text,list):\n",
    "        input_ids,token_type_ids,attention_mask=[],[],[]\n",
    "        for e in text:\n",
    "            output=tokenizer.encode_plus(e,max_length=max_len,pad_to_max_length=True,return_tensors=\"pt\")\n",
    "            input_ids.append(output[\"input_ids\"].squeeze(0))\n",
    "            token_type_ids.append(output[\"token_type_ids\"].squeeze(0))\n",
    "            attention_mask.append(output[\"attention_mask\"].squeeze(0))\n",
    "    else:\n",
    "        raise Exception('type error')\n",
    "    return torch.stack(input_ids).long(),torch.stack(token_type_ids).long(),torch.stack(attention_mask).long()        \n",
    "class RelDataset(Data.Dataset):\n",
    "    def __init__(self,examples):\n",
    "        self.input_ids=torch.stack([e['input_ids'] for e in examples]).long()\n",
    "        self.token_type_ids=torch.stack([e['token_type_ids'] for e in examples]).long()\n",
    "        self.attention_mask=torch.stack([e['attention_mask'] for e in examples]).long()\n",
    "        self.rel_label=torch.stack([e['rel_label'] for e in examples]).long()\n",
    "        self.postag=torch.stack([e['postag'] for e in examples]).long()\n",
    "        self.feature=torch.stack([e['feature'] for e in examples]).float()\n",
    "        self.token_vec=np.stack([np.array(e['token_vec']) for e in examples])\n",
    "        self.word_vec=np.stack([np.array(e['word_vec']) for e in examples])\n",
    "        self.word_mask=np.stack([np.array(e['word_mask']) for e in examples])\n",
    "        self.plan_label=np.stack([np.array(e['plan_label']) for e in examples])\n",
    "        self.token2docs=[e[\"token2doc\"] for e in examples]\n",
    "    def __len__(self):\n",
    "        return self.input_ids.shape[0]\n",
    "    def __getitem__(self,idx):\n",
    "        return self.input_ids[idx],self.attention_mask[idx],self.token_type_ids[idx],\\\n",
    "    self.rel_label[idx],self.postag[idx],self.feature[idx],self.token_vec[idx],self.word_vec[idx],\\\n",
    "    self.word_mask[idx],self.plan_label[idx],self.token2docs[idx]  \n",
    "class NerDataset(Data.Dataset):\n",
    "    def __init__(self,examples):\n",
    "        self.input_ids=torch.stack([e['input_ids'] for e in examples]).long()\n",
    "        self.token_type_ids=torch.stack([e['token_type_ids'] for e in examples]).long()\n",
    "        self.attention_mask=torch.stack([e['attention_mask'] for e in examples]).long()\n",
    "        self.rel_label=torch.stack([e['rel_label'] for e in examples]).long()\n",
    "        self.labels=torch.stack([e['labels'] for e in examples]).long()\n",
    "        self.postag=torch.stack([e['postag'] for e in examples]).long()\n",
    "        self.feature=torch.stack([e['feature'] for e in examples]).float()\n",
    "        self.plan_labels=torch.stack([e['plan_labels'] for e in examples]).long()\n",
    "        self.token2docs=[e[\"token2doc\"] for e in examples]\n",
    "    def __len__(self):\n",
    "        return self.input_ids.shape[0]\n",
    "    def __getitem__(self,idx):\n",
    "        return self.input_ids[idx],self.attention_mask[idx],self.token_type_ids[idx],\\\n",
    "               self.rel_label[idx],self.labels[idx],self.postag[idx],self.feature[idx],\\\n",
    "               self.plan_labels[idx],self.token2docs[idx]  \n",
    "import unicodedata\n",
    "def _is_whitespace(char):\n",
    "    \"\"\"Checks whether `chars` is a whitespace character.\"\"\"\n",
    "    # \\t, \\n, and \\r are technically contorl characters but we treat them\n",
    "    # as whitespace since they are generally considered as such.\n",
    "    if char == \" \" or char == \"\\t\" or char == \"\\n\" or char == \"\\r\":\n",
    "        return True\n",
    "    cat = unicodedata.category(char)\n",
    "    if cat == \"Zs\":\n",
    "        return True\n",
    "    return False\n",
    "def is_chinese_char(cp):\n",
    "    \"\"\"Checks whether CP is the codepoint of a CJK character.\"\"\"\n",
    "    # This defines a \"chinese character\" as anything in the CJK Unicode block:\n",
    "    #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)\n",
    "    #\n",
    "    # Note that the CJK Unicode block is NOT all Japanese and Korean characters,\n",
    "    # despite its name. The modern Korean Hangul alphabet is a different block,\n",
    "    # as is Japanese Hiragana and Katakana. Those alphabets are used to write\n",
    "    # space-separated words, so they are not treated specially and handled\n",
    "    # like the all of the other languages.\n",
    "    if (\n",
    "        (cp >= 0x4E00 and cp <= 0x9FFF)\n",
    "        or (cp >= 0x3400 and cp <= 0x4DBF)  #\n",
    "        or (cp >= 0x20000 and cp <= 0x2A6DF)  #\n",
    "        or (cp >= 0x2A700 and cp <= 0x2B73F)  #\n",
    "        or (cp >= 0x2B740 and cp <= 0x2B81F)  #\n",
    "        or (cp >= 0x2B820 and cp <= 0x2CEAF)  #\n",
    "        or (cp >= 0xF900 and cp <= 0xFAFF)\n",
    "        or (cp >= 0x2F800 and cp <= 0x2FA1F)  #\n",
    "    ):  #\n",
    "        return True\n",
    "\n",
    "    return False\n",
    "def is_punctuation(char):\n",
    "    \"\"\"Checks whether `chars` is a punctuation character.\"\"\"\n",
    "    cp = ord(char)\n",
    "    # We treat all non-letter/number ASCII as punctuation.\n",
    "    # Characters such as \"^\", \"$\", and \"`\" are not in the Unicode\n",
    "    # Punctuation class but we treat them as punctuation anyways, for\n",
    "    # consistency.\n",
    "    if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):\n",
    "        return True\n",
    "    cat = unicodedata.category(char)\n",
    "    if cat.startswith(\"P\"):\n",
    "        return True\n",
    "    return False\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "def cut_buff(info):\n",
    "    buff=\"\"\n",
    "    tokens=[]\n",
    "    for e in info:\n",
    "        if (e.isdigit() and (buff.isdigit() or buff=='')):\n",
    "            buff+=e\n",
    "        elif (e.isalpha() and (buff.isalpha() or buff=='')):\n",
    "            buff+=e\n",
    "        else:\n",
    "            if buff!=\"\":\n",
    "                tokens.append(buff)\n",
    "            buff=\"\"\n",
    "            if not (e.isdigit() or e.isalpha()):\n",
    "                tokens.append(e)\n",
    "            else:\n",
    "                buff+=e\n",
    "\n",
    "    if buff!=\"\":\n",
    "        tokens.append(buff)\n",
    "    return tokens\n",
    "def build_tfidf_svd_matrix(texts,n_output,tfidf_vec=None,svd=None):\n",
    "    corpus=[]\n",
    "    for text in tqdm(texts):\n",
    "#         print(text)\n",
    "        words=word_segment(str(text['text']))\n",
    "#         print(words)\n",
    "        use_words=[]\n",
    "        for word in words:\n",
    "            if word in stop_words:\n",
    "                continue\n",
    "            use_words.append(word)\n",
    "#         print(use_words)\n",
    "        corpus.append(\" \".join(use_words))\n",
    "    print(len(corpus))\n",
    "    print(corpus[0])\n",
    "    if tfidf_vec is None:\n",
    "        tfidf_vec=TfidfVectorizer()\n",
    "        tfidf_matrix=tfidf_vec.fit_transform(corpus)\n",
    "    else:\n",
    "        tfidf_matrix=tfidf_vec.transform(corpus)\n",
    "    if svd is None:\n",
    "        svd=TruncatedSVD(n_components=n_output,n_iter=7,random_state=42)\n",
    "        tf_idf_svd=svd.fit_transform(tfidf_matrix)\n",
    "    else:\n",
    "        tf_idf_svd=svd.transform(tfidf_matrix)\n",
    "    return tf_idf_svd,tfidf_vec,svd\n",
    "def word_segment(sentence):\n",
    "    words=jieba.cut(sentence)\n",
    "    return \",\".join(words).split(\",\")\n",
    "stop_words=set()\n",
    "def load_stopwords():\n",
    "    with open(\"./middle_data/stopwords.txt\",\"r\",encoding=\"UTF-8\") as r:\n",
    "        for line in r.readlines():\n",
    "            stop_words.add(line.strip())\n",
    "load_stopwords()\n",
    "def remove_stopwords(word_list):\n",
    "    res=[]\n",
    "    for word in word_lists:\n",
    "        if word not in stop_words:\n",
    "            res.append(word)\n",
    "    return ' '.join(res)\n",
    "def clean_text(string):\n",
    "    return string.replace(' ', '').replace('\\n', '').replace('\\u3000', '')\n",
    "def special_tokenize(text_raw,tokenizer):\n",
    "    sub_text = []\n",
    "    buff = \"\"\n",
    "    for char in text_raw:\n",
    "        if is_chinese_char(ord(char)) or is_punctuation(char):\n",
    "            if buff != \"\":\n",
    "                sub_text.extend(cut_buff(buff))\n",
    "                buff = \"\"\n",
    "            sub_text.append(char)\n",
    "\n",
    "        else:\n",
    "            buff += char\n",
    "    if buff != \"\":\n",
    "        sub_text.extend(cut_buff(buff))\n",
    "    tokens = []\n",
    "    text_tmp = ''\n",
    "    for (i, token) in enumerate(sub_text):\n",
    "        sub_tokens = tokenizer.tokenize(token)\n",
    "        for sub_token in sub_tokens:\n",
    "            tokens.append(sub_token)\n",
    "        else:\n",
    "            continue\n",
    "        break \n",
    "    return tokens\n",
    "def _convert_example_to_cls_record(example,\n",
    "                               tokenizer):\n",
    "    if example.__contains__('spo_list'):\n",
    "        spo_list = example['spo_list']\n",
    "    else:\n",
    "        spo_list = []\n",
    "    text_raw = example['text']\n",
    "    sub_text = []\n",
    "    buff = \"\"\n",
    "    for char in text_raw:\n",
    "        if is_chinese_char(ord(char)) or is_punctuation(char):\n",
    "            if buff != \"\":\n",
    "                sub_text.append(buff)\n",
    "                buff = \"\"\n",
    "            sub_text.append(char)\n",
    "        else:\n",
    "            buff += char\n",
    "    if buff != \"\":\n",
    "        sub_text.append(buff)\n",
    "    tok_to_orig_start_index = []\n",
    "    tok_to_orig_end_index = []\n",
    "    orig_to_tok_index = []\n",
    "    tokens = []\n",
    "    text_tmp = ''\n",
    "    for (i, token) in enumerate(sub_text):\n",
    "        orig_to_tok_index.append(len(tokens))\n",
    "        sub_tokens = tokenizer.tokenize(token)\n",
    "        text_tmp += token\n",
    "        for sub_token in sub_tokens:\n",
    "            tok_to_orig_start_index.append(len(text_tmp) - len(token))\n",
    "            tok_to_orig_end_index.append(len(text_tmp) - 1)\n",
    "            tokens.append(sub_token)\n",
    "        else:\n",
    "            continue\n",
    "        break\n",
    "    #  find all entities and tag them with corresponding \"B\"/\"I\" labels\n",
    "    labels_list=[]\n",
    "    rel_labels=[]\n",
    "    for spo in spo_list:\n",
    "        for spo_object in spo['object'].keys():\n",
    "            labels = [0\n",
    "                  for i in range(len(tokens))]  # initialize tag\n",
    "            if not spo['predicate'] in special_rels:\n",
    "                rel_label=rels2id[spo[\"predicate\"]]\n",
    "            else:\n",
    "                rel_label=rels2id[spo[\"predicate\"]+\"_\"+spo_object] \n",
    "            rel_labels.append(rel_label)\n",
    "\n",
    "    return tok_to_orig_start_index,tok_to_orig_end_index,orig_to_tok_index,tokens,rel_labels\n",
    "def _convert_example_to_record(example,\n",
    "                               tokenizer):\n",
    "    if example.__contains__('spo_list'):\n",
    "        spo_list = example['spo_list']\n",
    "    else:\n",
    "        spo_list = []\n",
    "    text_raw = example['text']\n",
    "    sub_text = []\n",
    "    buff = \"\"\n",
    "    for char in text_raw:\n",
    "        if is_chinese_char(ord(char)) or is_punctuation(char):\n",
    "            if buff != \"\":\n",
    "                sub_text.extend(cut_buff(buff))\n",
    "                buff = \"\"\n",
    "            sub_text.append(char)\n",
    "\n",
    "        else:\n",
    "            buff += char\n",
    "    if buff != \"\":\n",
    "        sub_text.extend(cut_buff(buff))\n",
    "    tok_to_orig_start_index = []\n",
    "    tok_to_orig_end_index = []\n",
    "    orig_to_tok_index = []\n",
    "    tokens = []\n",
    "    text_tmp = ''\n",
    "    for (i, token) in enumerate(sub_text):\n",
    "        orig_to_tok_index.append(len(tokens))\n",
    "        sub_tokens = tokenizer.tokenize(token)\n",
    "        text_tmp += token\n",
    "        for sub_token in sub_tokens:\n",
    "            tok_to_orig_start_index.append(len(text_tmp) - len(token))\n",
    "            tok_to_orig_end_index.append(len(text_tmp) - 1)\n",
    "            tokens.append(sub_token)\n",
    "        else:\n",
    "            continue\n",
    "        break\n",
    "    #  find all entities and tag them with corresponding \"B\"/\"I\" labels\n",
    "    labels_list=[[0\n",
    "              for i in range(len(tokens))] for i in range(len(id2rels))]\n",
    "    rel_labels=[]\n",
    "    for spo in spo_list:\n",
    "        for spo_object in spo['object'].keys():\n",
    "            if not spo['predicate'] in special_rels:\n",
    "                rel_label=rels2id[spo[\"predicate\"]]\n",
    "            else:\n",
    "                rel_label=rels2id[spo[\"predicate\"]+\"_\"+spo_object] \n",
    "            labels = labels_list[rel_label] #复杂类的不同part还是不会被归到一个，以后再讲\n",
    "            label_subject = label2ids['B-SUB']\n",
    "            label_object = label2ids['B-OBJ']\n",
    "            subject_sub_tokens = special_tokenize(spo['subject'],tokenizer)\n",
    "#             print(\"sub\",subject_sub_tokens)\n",
    "            object_sub_tokens = special_tokenize(spo['object'][\n",
    "                spo_object],tokenizer)\n",
    "            forbidden_index = None\n",
    "            if len(subject_sub_tokens) > len(object_sub_tokens):\n",
    "                for index in range(\n",
    "                        len(tokens) - len(subject_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            subject_sub_tokens)] == subject_sub_tokens:\n",
    "                        labels[index]=label_subject\n",
    "                        for i in range(len(subject_sub_tokens) - 1):\n",
    "                            labels[index + i + 1]=label_subject+1\n",
    "                        forbidden_index = index\n",
    "                        break\n",
    "\n",
    "                for index in range(\n",
    "                        len(tokens) - len(object_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            object_sub_tokens)] == object_sub_tokens:\n",
    "                        if forbidden_index is None:\n",
    "                            labels[index]=label_object\n",
    "                            for i in range(len(object_sub_tokens) - 1):\n",
    "                                labels[index + i + 1]=label_object+1\n",
    "                            break\n",
    "                        # check if labeled already\n",
    "                        elif index < forbidden_index or index >= forbidden_index + len(\n",
    "                                subject_sub_tokens):\n",
    "                            labels[index]=label_object\n",
    "                            for i in range(len(object_sub_tokens) - 1):\n",
    "                                labels[index + i + 1]=label_object+1\n",
    "                            break\n",
    "\n",
    "            else:\n",
    "                for index in range(\n",
    "                        len(tokens) - len(object_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            object_sub_tokens)] == object_sub_tokens:\n",
    "                        labels[index]=label_object\n",
    "                        for i in range(len(object_sub_tokens) - 1):\n",
    "                            labels[index + i + 1]=label_object+1\n",
    "                        forbidden_index = index\n",
    "                        break\n",
    "\n",
    "                for index in range(\n",
    "                        len(tokens) - len(subject_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            subject_sub_tokens)] == subject_sub_tokens:\n",
    "                        if forbidden_index is None:\n",
    "                            labels[index]=label_subject\n",
    "                            for i in range(len(subject_sub_tokens) - 1):\n",
    "                                labels[index + i + 1]=label_subject+1\n",
    "                            break\n",
    "                        elif index < forbidden_index or index >= forbidden_index + len(\n",
    "                                object_sub_tokens):\n",
    "                            labels[index]=label_subject\n",
    "                            for i in range(len(subject_sub_tokens) - 1):\n",
    "                                labels[index + i + 1]=label_subject+1\n",
    "                            break\n",
    "            labels_list[rel_label]=labels\n",
    "            if rel_label not in rel_labels:\n",
    "                rel_labels.append(rel_label)\n",
    "\n",
    "    return tok_to_orig_start_index,tok_to_orig_end_index,orig_to_tok_index,tokens,labels_list,rel_labels\n",
    "def _convert_example_to_record_new(example,\n",
    "                               tokenizer):\n",
    "    if example.__contains__('spo_list'):\n",
    "        spo_list = example['spo_list']\n",
    "    else:\n",
    "        spo_list = []\n",
    "    text_raw = example['text']\n",
    "    sub_text = []\n",
    "    buff = \"\"\n",
    "    for char in text_raw:\n",
    "        if is_chinese_char(ord(char)) or is_punctuation(char):\n",
    "            if buff != \"\":\n",
    "                sub_text.extend(cut_buff(buff))\n",
    "                buff = \"\"\n",
    "            sub_text.append(char)\n",
    "\n",
    "        else:\n",
    "            buff += char\n",
    "    if buff != \"\":\n",
    "        sub_text.extend(cut_buff(buff))\n",
    "    tok_to_orig_start_index = []\n",
    "    tok_to_orig_end_index = []\n",
    "    orig_to_tok_index = []\n",
    "    tokens = []\n",
    "    text_tmp = ''\n",
    "    for (i, token) in enumerate(sub_text):\n",
    "        orig_to_tok_index.append(len(tokens))\n",
    "        sub_tokens = tokenizer.tokenize(token)\n",
    "        text_tmp += token\n",
    "        for sub_token in sub_tokens:\n",
    "            tok_to_orig_start_index.append(len(text_tmp) - len(token))\n",
    "            tok_to_orig_end_index.append(len(text_tmp) - 1)\n",
    "            tokens.append(sub_token)\n",
    "        else:\n",
    "            continue\n",
    "        break\n",
    "    #  find all entities and tag them with corresponding \"B\"/\"I\" labels\n",
    "    labels_list=[[[0] * len(id2labels)\n",
    "              for i in range(len(tokens))] for i in range(len(id2rels))]\n",
    "    rel_labels=[]\n",
    "    for spo in spo_list:\n",
    "        for spo_object in spo['object'].keys():\n",
    "            if not spo['predicate'] in special_rels:\n",
    "                rel_label=rels2id[spo[\"predicate\"]]\n",
    "            else:\n",
    "                rel_label=rels2id[spo[\"predicate\"]+\"_\"+spo_object] \n",
    "            labels = labels_list[rel_label] #复杂类的不同part还是不会被归到一个，以后再讲\n",
    "            label_subject = new_label2ids['B-SUB']\n",
    "            label_object = new_label2ids['B-OBJ']\n",
    "            subject_sub_tokens = special_tokenize(spo['subject'],tokenizer)\n",
    "#             print(\"sub\",subject_sub_tokens)\n",
    "            object_sub_tokens = special_tokenize(spo['object'][\n",
    "                spo_object],tokenizer)\n",
    "            forbidden_index = None\n",
    "            if len(subject_sub_tokens) > len(object_sub_tokens):\n",
    "                for index in range(\n",
    "                        len(tokens) - len(subject_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            subject_sub_tokens)] == subject_sub_tokens:\n",
    "                        labels[index][label_subject]+=1\n",
    "                        labels[index + len(subject_sub_tokens)-1][label_subject+1]+=1\n",
    "                        forbidden_index = index\n",
    "                        break\n",
    "\n",
    "                for index in range(\n",
    "                        len(tokens) - len(object_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            object_sub_tokens)] == object_sub_tokens:\n",
    "                        if forbidden_index is None:\n",
    "                            labels[index][label_object]+=1\n",
    "                            labels[index + len(object_sub_tokens)-1][label_object+1]+=1\n",
    "                            break\n",
    "                        # check if labeled already\n",
    "                        elif index < forbidden_index or index >= forbidden_index + len(\n",
    "                                subject_sub_tokens):\n",
    "                            labels[index][label_object]+=1\n",
    "                            labels[index + len(object_sub_tokens)-1][label_object+1]+=1\n",
    "                            break\n",
    "\n",
    "            else:\n",
    "                for index in range(\n",
    "                        len(tokens) - len(object_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            object_sub_tokens)] == object_sub_tokens:\n",
    "                        labels[index][label_object]+=1\n",
    "                        labels[index + len(object_sub_tokens)-1][label_object+1]+=1\n",
    "                        forbidden_index = index\n",
    "                        break\n",
    "\n",
    "                for index in range(\n",
    "                        len(tokens) - len(subject_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            subject_sub_tokens)] == subject_sub_tokens:\n",
    "                        if forbidden_index is None:\n",
    "                            labels[index][label_subject]+=1\n",
    "                            labels[index + len(subject_sub_tokens)-1][label_subject+1]+=1\n",
    "                            break\n",
    "                        elif index < forbidden_index or index >= forbidden_index + len(\n",
    "                                object_sub_tokens):\n",
    "                            labels[index][label_subject]+=1\n",
    "                            labels[index + len(subject_sub_tokens)-1][label_subject+1]+=1\n",
    "                            break\n",
    "            labels_list[rel_label]=labels\n",
    "            if rel_label not in rel_labels:\n",
    "                rel_labels.append(rel_label)\n",
    "\n",
    "    return tok_to_orig_start_index,tok_to_orig_end_index,orig_to_tok_index,tokens,labels_list,rel_labels\n",
    "def create_cls_example(tokens,rel,token2doc,postag,tfidf_svd,word_vec,token_vec,plan_label,tokenizer):\n",
    "        word_vec=copy.deepcopy(word_vec)\n",
    "        token_vec=copy.deepcopy(token_vec)\n",
    "        if len(tokens)>max_len-2:\n",
    "                tokens=tokens[:(max_len-2)]\n",
    "                plan_label=plan_label[:(max_len-2)]\n",
    "                token2doc=[e[:(max_len-2)] if ind<2 else e for ind,e in enumerate(token2doc)]\n",
    "        tag=[postag[token2doc[0][idx]] for idx in range(len(tokens))]\n",
    "        tokenvec=[token_vec[token2doc[0][idx]] for idx in range(len(tokens))]\n",
    "        tag=[pos2id_BIO[tokenizer.cls_token]]+tag+[pos2id_BIO[tokenizer.sep_token]]\n",
    "        tokenvec=[word2id[tokenizer.cls_token]]+tokenvec+[word2id[tokenizer.sep_token]]\n",
    "        full_tokens=[tokenizer.cls_token]+tokens+[tokenizer.sep_token]\n",
    "        padding_O=[0]*len(id2kglabels)\n",
    "        padding_O[0]=1\n",
    "        full_plan_labels=[padding_O]+plan_label+[padding_O]\n",
    "        token_type_ids=[0]*len(full_tokens)\n",
    "        attention_mask=[1]*len(token_type_ids)\n",
    "        cur_len=len(full_tokens)\n",
    "        if cur_len<max_len:\n",
    "            full_tokens+=[tokenizer.pad_token]*(max_len-cur_len)\n",
    "            full_plan_labels+=[padding_O]*(max_len-cur_len)\n",
    "            attention_mask+=[0]*(max_len-cur_len)\n",
    "            token_type_ids+=[0]*(max_len-cur_len)\n",
    "            tag+=[pos2id_BIO[tokenizer.pad_token]]*(max_len-cur_len)\n",
    "            tokenvec+=[word2id[tokenizer.pad_token]]*(max_len-cur_len)\n",
    "\n",
    "        if len(word_vec)>max_len//2:\n",
    "            word_vec=word_vec[:max_len//2]\n",
    "        word_mask=[1]*len(word_vec)\n",
    "        if len(word_vec)<max_len//2:\n",
    "            word_mask+=[0]*(max_len//2-len(word_vec))\n",
    "            word_vec+=[word2id[tokenizer.pad_token]]*(max_len//2-len(word_vec))\n",
    "\n",
    "        full_ids=tokenizer.convert_tokens_to_ids(full_tokens)\n",
    "        if len(rel)>0:\n",
    "            example={\"input_ids\":torch.tensor(full_ids,dtype=torch.long),\"token_type_ids\":torch.tensor(token_type_ids,dtype=torch.long),\\\n",
    "                    \"attention_mask\":torch.tensor(attention_mask,dtype=torch.long),\n",
    "                    \"rel_label\":(F.one_hot(torch.tensor(rel),len(id2rels)).sum(dim=0)!=0).long(),\n",
    "                     \"postag\":torch.tensor(tag).long(),\"feature\":torch.tensor(tfidf_svd).float(),\\\n",
    "                     \"token_vec\":tokenvec,\"word_vec\":word_vec,\"word_mask\":word_mask,\"plan_label\":full_plan_labels,\"token2doc\":token2doc}\n",
    "        else:\n",
    "            example={\"input_ids\":torch.tensor(full_ids,dtype=torch.long),\"token_type_ids\":torch.tensor(token_type_ids,dtype=torch.long),\\\n",
    "                    \"attention_mask\":torch.tensor(attention_mask,dtype=torch.long),\n",
    "                     \"rel_label\":torch.zeros(len(id2rels)).long(),\n",
    "                     \"postag\":torch.tensor(tag).long(),\"feature\":torch.tensor(tfidf_svd).float(),\\\n",
    "                     \"token_vec\":tokenvec,\"word_vec\":word_vec,\"word_mask\":word_mask,\"plan_label\":full_plan_labels,\"token2doc\":token2doc}  \n",
    "        return example\n",
    "def create_example(tokens,rel,labels,token2doc,tokenizer,rel_text,tfidf_svd,postag,rel_postag,rel_token2doc):\n",
    "        tag=[postag[token2doc[0][idx]] for idx in range(len(tokens))]\n",
    "        tag=[pos2id_BIO[tokenizer.cls_token]]+tag+[pos2id_BIO[tokenizer.sep_token]]\n",
    "        rel_tag=[rel_postag[rel_token2doc[idx]] for idx in range(len(rel_text))]\n",
    "        rel_tag=rel_tag+[pos2id_BIO[tokenizer.sep_token]]\n",
    "        tag=tag+rel_tag\n",
    "        second_token=rel_text\n",
    "        full_tokens=[[tokenizer.cls_token]+tokens+[tokenizer.sep_token],second_token+[tokenizer.sep_token]]\n",
    "        full_labels=[[label2ids[tokenizer.cls_token]]+labels+[label2ids[tokenizer.sep_token]],[label2ids[\"[category]\"]]*len(second_token)+[label2ids[tokenizer.sep_token]]]\n",
    "        token_type_ids=[0]*len(full_tokens[0])+[1]*len(full_tokens[1])\n",
    "        attention_mask=[1]*len(token_type_ids)\n",
    "        full_tokens=full_tokens[0]+full_tokens[1]\n",
    "        full_labels=full_labels[0]+full_labels[1]\n",
    "        cur_len=len(full_labels)\n",
    "        if cur_len<max_len:\n",
    "            full_tokens+=[tokenizer.pad_token]*(max_len-cur_len)\n",
    "            full_labels+=[label2ids[tokenizer.pad_token]]*(max_len-cur_len)\n",
    "            attention_mask+=[0]*(max_len-cur_len)\n",
    "            token_type_ids+=[0]*(max_len-cur_len)\n",
    "            tag+=[pos2id_BIO[tokenizer.pad_token]]*(max_len-cur_len)\n",
    "        full_ids=tokenizer.convert_tokens_to_ids(full_tokens)\n",
    "        example={\"input_ids\":torch.tensor(full_ids,dtype=torch.long),\"token_type_ids\":torch.tensor(token_type_ids,dtype=torch.long),\\\n",
    "                \"attention_mask\":torch.tensor(attention_mask,dtype=torch.long),\"labels\":torch.tensor(full_labels,dtype=torch.long),\n",
    "                \"rel_label\":F.one_hot(torch.tensor(rel),num_classes=len(id2rels)),\"postag\":torch.tensor(tag).long(),\"feature\":torch.tensor(tfidf_svd).float(),\\\n",
    "                 \"token2doc\":token2doc}\n",
    "        return example\n",
    "def create_example_new(tokens,rel,labels,token2doc,tokenizer,rel_text,tfidf_svd,postag,rel_postag,rel_token2doc,plan_labels):\n",
    "        tag=[postag[token2doc[0][idx]] for idx in range(len(tokens))]\n",
    "        tag=[pos2id_BIO[tokenizer.cls_token]]+tag+[pos2id_BIO[tokenizer.sep_token]]\n",
    "        rel_tag=[rel_postag[rel_token2doc[idx]] for idx in range(len(rel_text))]\n",
    "        rel_tag=rel_tag+[pos2id_BIO[tokenizer.sep_token]]\n",
    "        tag=tag+rel_tag\n",
    "        second_token=rel_text\n",
    "        full_tokens=[[tokenizer.cls_token]+tokens+[tokenizer.sep_token],second_token+[tokenizer.sep_token]]\n",
    "        full_labels=[[padding_CLS]+labels+[padding_SEP],[padding_category]*len(second_token)+[padding_SEP]]\n",
    "        full_plan_labels=[[padding_CLS]+plan_labels+[padding_SEP],[padding_category]*len(second_token)+[padding_SEP]]\n",
    "        token_type_ids=[0]*len(full_tokens[0])+[1]*len(full_tokens[1])\n",
    "        attention_mask=[1]*len(token_type_ids)\n",
    "        full_tokens=full_tokens[0]+full_tokens[1]\n",
    "        full_labels=full_labels[0]+full_labels[1]\n",
    "        full_plan_labels=full_plan_labels[0]+full_plan_labels[1]\n",
    "        cur_len=len(full_labels)\n",
    "        if cur_len<max_len:\n",
    "            full_tokens+=[tokenizer.pad_token]*(max_len-cur_len)\n",
    "            full_labels+=[padding_PAD]*(max_len-cur_len)\n",
    "            full_plan_labels+=[padding_PAD]*(max_len-cur_len)\n",
    "            attention_mask+=[0]*(max_len-cur_len)\n",
    "            token_type_ids+=[0]*(max_len-cur_len)\n",
    "            tag+=[pos2id_BIO[tokenizer.pad_token]]*(max_len-cur_len)\n",
    "        full_ids=tokenizer.convert_tokens_to_ids(full_tokens)\n",
    "        example={\"input_ids\":torch.tensor(full_ids,dtype=torch.long),\"token_type_ids\":torch.tensor(token_type_ids,dtype=torch.long),\\\n",
    "                 \"attention_mask\":torch.tensor(attention_mask,dtype=torch.long),\"labels\":torch.tensor(full_labels,dtype=torch.long),\n",
    "                \"rel_label\":F.one_hot(torch.tensor(rel),num_classes=len(id2rels)),\"postag\":torch.tensor(tag).long(),\"feature\":torch.tensor(tfidf_svd).float(),\\\n",
    "                 \"plan_labels\":torch.tensor(full_plan_labels,dtype=torch.long),\"token2doc\":token2doc}\n",
    "        return example\n",
    "def index_token(text_raw):\n",
    "    sub_text = []\n",
    "    buff = \"\"\n",
    "    for char in text_raw:\n",
    "        if is_chinese_char(ord(char)) or is_punctuation(char):\n",
    "            if buff != \"\":\n",
    "                sub_text.append(buff)\n",
    "                buff = \"\"\n",
    "            sub_text.append(char)\n",
    "        else:\n",
    "            buff += char\n",
    "    if buff != \"\":\n",
    "        sub_text.append(buff)\n",
    "    tok_to_orig_start_index = []\n",
    "    tok_to_orig_end_index = []\n",
    "    orig_to_tok_index = []\n",
    "    tokens = []\n",
    "    text_tmp = ''\n",
    "    for (i, token) in enumerate(sub_text):\n",
    "        orig_to_tok_index.append(len(tokens))\n",
    "        sub_tokens = tokenizer.tokenize(token)\n",
    "        text_tmp += token\n",
    "        for sub_token in sub_tokens:\n",
    "            tok_to_orig_start_index.append(len(text_tmp) - len(token))\n",
    "            tok_to_orig_end_index.append(len(text_tmp) - 1)\n",
    "            tokens.append(sub_token)\n",
    "        else:\n",
    "            continue\n",
    "        break\n",
    "    return tok_to_orig_start_index,tok_to_orig_end_index,orig_to_tok_index\n",
    "def find_subs_obs(token_label_pred,text):\n",
    "    subs=[]\n",
    "    sub_offsets=[]\n",
    "    obs=[]\n",
    "    ob_offsets=[]\n",
    "    index=0\n",
    "    while index<token_label_pred.shape[0]:\n",
    "        if token_label_pred[index]==3:\n",
    "            subs.append(text[index])\n",
    "            index+=1\n",
    "            while(index<token_label_pred.shape[0] and (token_label_pred[index]==4 or \\\n",
    "                                                       (token_label_pred[index]==3 and token_label_pred[index-1]==3))):\n",
    "                subs[-1]+=(text[index])\n",
    "                index+=1\n",
    "            sub_offsets.append([index-len(subs[-1]),index])\n",
    "            index-=1\n",
    "        if index<token_label_pred.shape[0] and token_label_pred[index]==1:\n",
    "            obs.append(text[index])\n",
    "            index+=1\n",
    "            while(index<token_label_pred.shape[0] and (token_label_pred[index]==2 or \\\n",
    "                                                       (token_label_pred[index]==1 and token_label_pred[index-1]==1))):\n",
    "                obs[-1]+=(text[index])\n",
    "                index+=1\n",
    "            ob_offsets.append([index-len(obs[-1]),index])\n",
    "            index-=1\n",
    "        index+=1\n",
    "    if len(subs)==0:\n",
    "        subs.append('')\n",
    "        sub_offsets.append([0,0])\n",
    "    if len(obs)==0:\n",
    "        obs.append('')\n",
    "        ob_offsets.append([0,0])\n",
    "    return subs,obs,sub_offsets,ob_offsets\n",
    "def _convert_example_to_cls_plan_labels(example,\n",
    "                               tokenizer):\n",
    "    if example.__contains__('plan_spo_list'):\n",
    "        spo_list = example['plan_spo_list']\n",
    "    else:\n",
    "        spo_list = []\n",
    "    text_raw = example['text']\n",
    "    sub_text = []\n",
    "    buff = \"\"\n",
    "    for char in text_raw:\n",
    "        if is_chinese_char(ord(char)) or is_punctuation(char):\n",
    "            if buff != \"\":\n",
    "                sub_text.append(buff)\n",
    "                buff = \"\"\n",
    "            sub_text.append(char)\n",
    "        else:\n",
    "            buff += char\n",
    "    if buff != \"\":\n",
    "        sub_text.append(buff)\n",
    "    tok_to_orig_start_index = []\n",
    "    tok_to_orig_end_index = []\n",
    "    orig_to_tok_index = []\n",
    "    tokens = []\n",
    "    text_tmp = ''\n",
    "    for (i, token) in enumerate(sub_text):\n",
    "        orig_to_tok_index.append(len(tokens))\n",
    "        sub_tokens = tokenizer.tokenize(token)\n",
    "        text_tmp += token\n",
    "        for sub_token in sub_tokens:\n",
    "            tok_to_orig_start_index.append(len(text_tmp) - len(token))\n",
    "            tok_to_orig_end_index.append(len(text_tmp) - 1)\n",
    "            tokens.append(sub_token)\n",
    "        else:\n",
    "            continue\n",
    "        break\n",
    "    #  find all entities and tag them with corresponding \"B\"/\"I\" labels\n",
    "    labels = [[0] * len(id2kglabels)\n",
    "              for i in range(len(tokens))]  # initialize tag\n",
    "    for spo in spo_list:\n",
    "            label_subject = kglabels2id['B-'+spo['predicate']+'-SUB']\n",
    "            label_object = kglabels2id['B-'+spo['predicate']+'-OB']\n",
    "            subject_sub_tokens = tokenizer.tokenize(spo['subject'])\n",
    "            object_sub_tokens = tokenizer.tokenize(spo['object'])\n",
    "            forbidden_index = None\n",
    "            if len(subject_sub_tokens) > len(object_sub_tokens):\n",
    "                for index in range(\n",
    "                        len(tokens) - len(subject_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            subject_sub_tokens)] == subject_sub_tokens:\n",
    "                        labels[index][label_subject]=1\n",
    "                        for i in range(len(subject_sub_tokens) - 1):\n",
    "                            labels[index+i+1][kglabels2id['I']]=1\n",
    "                        forbidden_index = index\n",
    "                        break\n",
    "\n",
    "                for index in range(\n",
    "                        len(tokens) - len(object_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            object_sub_tokens)] == object_sub_tokens:\n",
    "                        if forbidden_index is None:\n",
    "                            labels[index][label_object]=1\n",
    "                            for i in range(len(object_sub_tokens) - 1):\n",
    "                                labels[index + i + 1][kglabels2id['I']]=1\n",
    "                            break\n",
    "                        # check if labeled already\n",
    "                        elif index < forbidden_index or index >= forbidden_index + len(\n",
    "                                subject_sub_tokens):\n",
    "                            labels[index][label_object]=1\n",
    "                            for i in range(len(object_sub_tokens) - 1):\n",
    "                                labels[index + i + 1][kglabels2id['I']]=1\n",
    "                            break\n",
    "\n",
    "            else:\n",
    "                for index in range(\n",
    "                        len(tokens) - len(object_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            object_sub_tokens)] == object_sub_tokens:\n",
    "                        labels[index][label_object]=1\n",
    "                        for i in range(len(object_sub_tokens) - 1):\n",
    "                            labels[index + i + 1][kglabels2id['I']]=1\n",
    "                        forbidden_index = index\n",
    "                        break\n",
    "\n",
    "                for index in range(\n",
    "                        len(tokens) - len(subject_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            subject_sub_tokens)] == subject_sub_tokens:\n",
    "                        if forbidden_index is None:\n",
    "                            labels[index][label_subject]=1\n",
    "                            for i in range(len(subject_sub_tokens) - 1):\n",
    "                                labels[index + i + 1][kglabels2id['I']]=1\n",
    "                            break\n",
    "                        elif index < forbidden_index or index >= forbidden_index + len(\n",
    "                                object_sub_tokens):\n",
    "                            labels[index][label_subject]=1\n",
    "                            for i in range(len(subject_sub_tokens) - 1):\n",
    "                                labels[index + i + 1][kglabels2id['I']]=1\n",
    "                            break\n",
    "    return labels\n",
    "def _convert_example_to_plan_labels(example,\n",
    "                               tokenizer):\n",
    "    if example.__contains__('plan_spo_list'):\n",
    "        spo_list = example['plan_spo_list']\n",
    "    else:\n",
    "        spo_list = []\n",
    "    text_raw = example['text']\n",
    "    sub_text = []\n",
    "    buff = \"\"\n",
    "    for char in text_raw:\n",
    "        if is_chinese_char(ord(char)) or is_punctuation(char):\n",
    "            if buff != \"\":\n",
    "                sub_text.extend(cut_buff(buff))\n",
    "                buff = \"\"\n",
    "            sub_text.append(char)\n",
    "\n",
    "        else:\n",
    "            buff += char\n",
    "    if buff != \"\":\n",
    "        sub_text.extend(cut_buff(buff))\n",
    "    tok_to_orig_start_index = []\n",
    "    tok_to_orig_end_index = []\n",
    "    orig_to_tok_index = []\n",
    "    tokens = []\n",
    "    text_tmp = ''\n",
    "    for (i, token) in enumerate(sub_text):\n",
    "        orig_to_tok_index.append(len(tokens))\n",
    "        sub_tokens = tokenizer.tokenize(token)\n",
    "        text_tmp += token\n",
    "        for sub_token in sub_tokens:\n",
    "            tok_to_orig_start_index.append(len(text_tmp) - len(token))\n",
    "            tok_to_orig_end_index.append(len(text_tmp) - 1)\n",
    "            tokens.append(sub_token)\n",
    "        else:\n",
    "            continue\n",
    "        break\n",
    "    #  find all entities and tag them with corresponding \"B\"/\"I\" labels\n",
    "    labels = [[0] * len(new_id2kglabels)\n",
    "              for i in range(len(tokens))]  # initialize tag\n",
    "    for spo in spo_list:\n",
    "            label_subject = kglabels2id['B-'+spo['predicate']+'-SUB']\n",
    "            label_object = kglabels2id['B-'+spo['predicate']+'-OB']\n",
    "            subject_sub_tokens = special_tokenize(spo['subject'],tokenizer)\n",
    "            object_sub_tokens = special_tokenize(spo['object'],tokenizer)\n",
    "            forbidden_index = None\n",
    "            if len(subject_sub_tokens) > len(object_sub_tokens):\n",
    "                for index in range(\n",
    "                        len(tokens) - len(subject_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            subject_sub_tokens)] == subject_sub_tokens:\n",
    "                        labels[index][label_subject]=1\n",
    "                        for i in range(len(subject_sub_tokens) - 1):\n",
    "                            labels[index+i+1][kglabels2id['I']]=1\n",
    "                        forbidden_index = index\n",
    "                        break\n",
    "\n",
    "                for index in range(\n",
    "                        len(tokens) - len(object_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            object_sub_tokens)] == object_sub_tokens:\n",
    "                        if forbidden_index is None:\n",
    "                            labels[index][label_object]=1\n",
    "                            for i in range(len(object_sub_tokens) - 1):\n",
    "                                labels[index + i + 1][kglabels2id['I']]=1\n",
    "                            break\n",
    "                        # check if labeled already\n",
    "                        elif index < forbidden_index or index >= forbidden_index + len(\n",
    "                                subject_sub_tokens):\n",
    "                            labels[index][label_object]=1\n",
    "                            for i in range(len(object_sub_tokens) - 1):\n",
    "                                labels[index + i + 1][kglabels2id['I']]=1\n",
    "                            break\n",
    "\n",
    "            else:\n",
    "                for index in range(\n",
    "                        len(tokens) - len(object_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            object_sub_tokens)] == object_sub_tokens:\n",
    "                        labels[index][label_object]=1\n",
    "                        for i in range(len(object_sub_tokens) - 1):\n",
    "                            labels[index + i + 1][kglabels2id['I']]=1\n",
    "                        forbidden_index = index\n",
    "                        break\n",
    "\n",
    "                for index in range(\n",
    "                        len(tokens) - len(subject_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            subject_sub_tokens)] == subject_sub_tokens:\n",
    "                        if forbidden_index is None:\n",
    "                            labels[index][label_subject]=1\n",
    "                            for i in range(len(subject_sub_tokens) - 1):\n",
    "                                labels[index + i + 1][kglabels2id['I']]=1\n",
    "                            break\n",
    "                        elif index < forbidden_index or index >= forbidden_index + len(\n",
    "                                object_sub_tokens):\n",
    "                            labels[index][label_subject]=1\n",
    "                            for i in range(len(subject_sub_tokens) - 1):\n",
    "                                labels[index + i + 1][kglabels2id['I']]=1\n",
    "                            break\n",
    "    return labels\n",
    "def _convert_example_to_plan_labels_new(example,\n",
    "                               tokenizer):\n",
    "    if example.__contains__('plan_spo_list'):\n",
    "        spo_list = example['plan_spo_list']\n",
    "    else:\n",
    "        spo_list = []\n",
    "    text_raw = example['text']\n",
    "    sub_text = []\n",
    "    buff = \"\"\n",
    "    for char in text_raw:\n",
    "        if is_chinese_char(ord(char)) or is_punctuation(char):\n",
    "            if buff != \"\":\n",
    "                sub_text.extend(cut_buff(buff))\n",
    "                buff = \"\"\n",
    "            sub_text.append(char)\n",
    "\n",
    "        else:\n",
    "            buff += char\n",
    "    if buff != \"\":\n",
    "        sub_text.extend(cut_buff(buff))\n",
    "    tok_to_orig_start_index = []\n",
    "    tok_to_orig_end_index = []\n",
    "    orig_to_tok_index = []\n",
    "    tokens = []\n",
    "    text_tmp = ''\n",
    "    for (i, token) in enumerate(sub_text):\n",
    "        orig_to_tok_index.append(len(tokens))\n",
    "        sub_tokens = tokenizer.tokenize(token)\n",
    "        text_tmp += token\n",
    "        for sub_token in sub_tokens:\n",
    "            tok_to_orig_start_index.append(len(text_tmp) - len(token))\n",
    "            tok_to_orig_end_index.append(len(text_tmp) - 1)\n",
    "            tokens.append(sub_token)\n",
    "        else:\n",
    "            continue\n",
    "        break\n",
    "    #  find all entities and tag them with corresponding \"B\"/\"I\" labels\n",
    "    labels = [[0] * len(new_id2kglabels)\n",
    "              for i in range(len(tokens))]  # initialize tag\n",
    "    for spo in spo_list:\n",
    "            label_subject = new_kglabels2id['B-'+spo['predicate']+'-SUB']\n",
    "            label_object = new_kglabels2id['B-'+spo['predicate']+'-OB']\n",
    "            subject_sub_tokens = special_tokenize(spo['subject'],tokenizer)\n",
    "            object_sub_tokens = special_tokenize(spo['object'],tokenizer)\n",
    "            forbidden_index = None\n",
    "            if len(subject_sub_tokens) > len(object_sub_tokens):\n",
    "                for index in range(\n",
    "                        len(tokens) - len(subject_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            subject_sub_tokens)] == subject_sub_tokens:\n",
    "                        labels[index][label_subject]+=1\n",
    "                        labels[index+len(subject_sub_tokens)-1][new_kglabels2id['E-'+spo['predicate']+'-SUB']]+=1\n",
    "                        forbidden_index = index\n",
    "                        break\n",
    "\n",
    "                for index in range(\n",
    "                        len(tokens) - len(object_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            object_sub_tokens)] == object_sub_tokens:\n",
    "                        if forbidden_index is None:\n",
    "                            labels[index][label_object]+=1\n",
    "                            labels[index+len(object_sub_tokens)-1][new_kglabels2id['E-'+spo['predicate']+'-OB']]+=1\n",
    "                            break\n",
    "                        # check if labeled already\n",
    "                        elif index < forbidden_index or index >= forbidden_index + len(\n",
    "                                subject_sub_tokens):\n",
    "                            labels[index][label_object]+=1\n",
    "                            labels[index+len(object_sub_tokens)-1][new_kglabels2id['E-'+spo['predicate']+'-OB']]+=1\n",
    "                            break\n",
    "\n",
    "            else:\n",
    "                for index in range(\n",
    "                        len(tokens) - len(object_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            object_sub_tokens)] == object_sub_tokens:\n",
    "                        labels[index][label_object]+=1\n",
    "                        labels[index+len(object_sub_tokens)-1][new_kglabels2id['E-'+spo['predicate']+'-OB']]+=1\n",
    "                        forbidden_index = index\n",
    "                        break\n",
    "\n",
    "                for index in range(\n",
    "                        len(tokens) - len(subject_sub_tokens) + 1):\n",
    "                    if tokens[index:index + len(\n",
    "                            subject_sub_tokens)] == subject_sub_tokens:\n",
    "                        if forbidden_index is None:\n",
    "                            labels[index][label_subject]+=1\n",
    "                            labels[index+len(subject_sub_tokens)-1][new_kglabels2id['E-'+spo['predicate']+'-SUB']]+=1\n",
    "                            break\n",
    "                        elif index < forbidden_index or index >= forbidden_index + len(\n",
    "                                object_sub_tokens):\n",
    "                            labels[index][label_subject]+=1\n",
    "                            labels[index+len(subject_sub_tokens)-1][new_kglabels2id['E-'+spo['predicate']+'-SUB']]+=1\n",
    "                            break\n",
    "    return labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/11/2020 16:21:00 - INFO - transformers.tokenization_utils -   Model name './roberta-zh-wwm-pytorch/vocab.txt' not found in model shortcut name list (bert-base-uncased, bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, bert-base-multilingual-cased, bert-base-chinese, bert-base-german-cased, bert-large-uncased-whole-word-masking, bert-large-cased-whole-word-masking, bert-large-uncased-whole-word-masking-finetuned-squad, bert-large-cased-whole-word-masking-finetuned-squad, bert-base-cased-finetuned-mrpc, bert-base-german-dbmdz-cased, bert-base-german-dbmdz-uncased, bert-base-finnish-cased-v1, bert-base-finnish-uncased-v1, bert-base-dutch-cased). Assuming './roberta-zh-wwm-pytorch/vocab.txt' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
      "05/11/2020 16:21:00 - WARNING - transformers.tokenization_utils -   Calling BertTokenizer.from_pretrained() with the path to a single file or url is deprecated\n",
      "05/11/2020 16:21:00 - INFO - transformers.tokenization_utils -   loading file ./roberta-zh-wwm-pytorch/vocab.txt\n"
     ]
    }
   ],
   "source": [
    "\n",
    "do_lower_case=True\n",
    "max_len=256\n",
    "# bert_dir=\"./bert-pytorch-chinese/\"\n",
    "# vocab=\"bert-base-chinese-vocab.txt\"\n",
    "# config_file=\"bert_config.json\"\n",
    "bert_dir=\"./roberta-zh-wwm-pytorch/\"\n",
    "vocab=\"vocab.txt\"\n",
    "config_file=\"bert_config.json\"\n",
    "tokenizer=BertTokenizer.from_pretrained(os.path.join(bert_dir,vocab),do_lower_case=do_lower_case)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "code_folding": [
     12,
     22,
     27,
     33,
     40,
     47,
     57,
     78,
     87,
     118
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading dict...\n",
      "loading...\n"
     ]
    }
   ],
   "source": [
    "text_data=[]\n",
    "with open(\"./dataset/train_data/new_train_data.json\",\"r\") as r:\n",
    "    raw_data=r.readlines()\n",
    "    for d in raw_data:\n",
    "        text_data.append(json.loads(d))\n",
    "test_text_data=[]\n",
    "with open(\"./dataset/test1_data/new_test1_data.json\",\"r\") as r:\n",
    "    raw_data=r.readlines()\n",
    "    for d in raw_data:\n",
    "        test_text_data.append(json.loads(d))\n",
    "schema=[]\n",
    "with open(\"./dataset/schema.json\",\"r\") as r:\n",
    "    raw_schema=r.readlines()\n",
    "    for d in raw_schema:\n",
    "        schema.append(json.loads(d))\n",
    "rels=set()\n",
    "special_rels=set()\n",
    "for e in schema:\n",
    "        if len(e['object_type'].keys())==1:\n",
    "            rels.add(e[\"predicate\"])\n",
    "        else:\n",
    "            special_rels.add(e[\"predicate\"])\n",
    "            for key in e['object_type'].keys():\n",
    "                rels.add(e['predicate']+\"_\"+key)\n",
    "if not os.path.exists(\"./dataset/dict.pk\"):\n",
    "    special_rels=list(special_rels)\n",
    "    id2rels=list(rels)\n",
    "    rels2id=dict([(rel,idx) for idx,rel in enumerate(id2rels)])\n",
    "    id2labels=[\"O\",\"B-OBJ\",\"I-OBJ\",\"B-SUB\",\"I-SUB\",\"[category]\",\"[SEP]\",\"[CLS]\",\"[PAD]\"]\n",
    "    label2ids=dict([ (label,idx) for idx,label in enumerate(id2labels)])\n",
    "    pk.dump([special_rels,id2rels,rels2id,id2labels,label2ids],open(\"./dataset/dict.pk\",\"wb\"))\n",
    "else:\n",
    "    print(\"loading dict...\")\n",
    "    special_rels,id2rels,rels2id,id2labels,label2ids=pk.load(open(\"./dataset/dict.pk\",\"rb\"))\n",
    "id2kglabels=['O','I']+['B-'+e+\"-SUB\" for e in id2rels]+['B-'+e+\"-OB\" for e in id2rels]\n",
    "kglabels2id=dict([ (label,idx) for idx,label in enumerate(id2kglabels)])\n",
    "new_id2labels=[\"O\",\"B-OBJ\",\"E-OBJ\",\"B-SUB\",\"E-SUB\",\"[category]\",\"[SEP]\",\"[CLS]\",\"[PAD]\"]\n",
    "new_label2ids=dict([ (label,idx) for idx,label in enumerate(new_id2labels)])\n",
    "new_id2kglabels=['O']+['B-'+e+\"-SUB\" for e in id2rels]+['B-'+e+\"-OB\" for e in id2rels]+['E-'+e+\"-SUB\" for e in id2rels]+['E-'+e+\"-OB\" for e in id2rels]\n",
    "new_kglabels2id=dict([ (label,idx) for idx,label in enumerate(new_id2kglabels)])\n",
    "id2reltype=[[] for i in range(len(id2rels))]\n",
    "for e in schema:\n",
    "    if len(e['object_type'].keys())==1:\n",
    "        rel=e[\"predicate\"]\n",
    "        ids=rels2id[rel]\n",
    "        id2reltype[ids].append(e)\n",
    "    else:\n",
    "        for key in e['object_type'].keys():\n",
    "            rel=e['predicate']+\"_\"+key\n",
    "            ids=rels2id[rel]\n",
    "            temp_e=copy.deepcopy(e)\n",
    "            poped_keys=[]\n",
    "            for k in temp_e['object_type'].keys():\n",
    "                if k!=key:\n",
    "                    poped_keys.append(k)\n",
    "            for k in poped_keys:\n",
    "                 temp_e['object_type'].pop(k)\n",
    "            id2reltype[ids].append(temp_e)\n",
    "id2schema=[e[0] for e in id2reltype]\n",
    "id2rel_text=[[] for i in range(len(id2rels))]\n",
    "id2rel_rawtext=[[] for i in range(len(id2rels))]\n",
    "id2rel_token2text=[[] for i in range(len(id2rels))]\n",
    "for rel in range(len(id2rels)):\n",
    "    if id2rels[rel].split(\"_\")[0] not in special_rels:\n",
    "        cls_text=id2schema[rel]['subject_type']+\",\"+id2schema[rel]['predicate']+\",\"+id2schema[rel]['object_type']['@value']\n",
    "    else:\n",
    "        cls_text=id2schema[rel]['subject_type']+\",\"+id2schema[rel]['predicate']+\",\"+id2schema[rel]['object_type'][id2rels[rel].split(\"_\")[1]]\n",
    "    id2rel_text[rel]=tokenizer.tokenize(cls_text)\n",
    "    id2rel_rawtext[rel]=cls_text\n",
    "    id2rel_token2text[rel]=index_token(cls_text)[0]\n",
    "    assert len(id2rel_token2text[rel])==len(id2rel_text[rel])\n",
    "if not os.path.exists(\"./middle_data/rel_data_postag.pk\"):\n",
    "    jieba.enable_paddle() \n",
    "    jieba.enable_parallel(8)\n",
    "    rel_cut_words=[]\n",
    "    rel_cut_tags=[]\n",
    "    for idx in tqdm(range(len(id2rel_rawtext))):\n",
    "        words = pseg.lcut(id2rel_rawtext[idx],use_paddle=True) #jieba默认模式\n",
    "        new_words=[w for w,t in words]\n",
    "        new_tags=[t for w,t in words]\n",
    "        rel_cut_words.append([idx,new_words])\n",
    "        rel_cut_tags.append([idx,new_tags])\n",
    "    rel_cut_words=[e[1] for e in sorted(rel_cut_words,key=lambda x:x[0])]\n",
    "    rel_cut_tags=[e[1] for e in sorted(rel_cut_tags,key=lambda x:x[0])]\n",
    "    rel_data_postag=[]\n",
    "    for idx in tqdm(range(len(id2rel_rawtext))):\n",
    "        assert len(id2rel_rawtext[idx].strip())==len(\"\".join(rel_cut_words[idx]))\n",
    "        indexs=[]\n",
    "        cur_length=0\n",
    "        for e in rel_cut_words[idx]:\n",
    "            indexs.append([cur_length,cur_length+len(e)-1])\n",
    "            cur_length+=len(e)\n",
    "        pos_label=np.zeros(len(id2rel_rawtext[idx])).astype(np.int8)\n",
    "        for i,(b,e) in enumerate(indexs):\n",
    "            assert (id2rel_rawtext[idx][b]==rel_cut_words[idx][i][0] or _is_whitespace(id2rel_rawtext[idx].strip()[b])\\\n",
    "                   or _is_whitespace(rel_cut_words[idx][i][0])) \\\n",
    "                    and (id2rel_rawtext[idx].strip()[e]==rel_cut_words[idx][i][-1] \\\n",
    "                         or _is_whitespace(id2rel_rawtext[idx].strip()[e])  or _is_whitespace(rel_cut_words[idx][i][-1]))\n",
    "            pos_label[b+1:e+1]=pos2id_BIO['I-'+rel_cut_tags[idx][i]]\n",
    "            pos_label[b]=pos2id_BIO['B-'+rel_cut_tags[idx][i]]\n",
    "        rel_data_postag.append(pos_label)\n",
    "    pk.dump(rel_data_postag,open(\"./middle_data/rel_data_postag.pk\",\"wb\"))\n",
    "else:\n",
    "    print(\"loading...\")\n",
    "    rel_data_postag=pk.load(open(\"./middle_data/rel_data_postag.pk\",\"rb\"))\n",
    "special_major_idx=[2,4,22,32,54]\n",
    "special_affilate_idx=[[] for i in range(len(id2rels))]\n",
    "special_affilate_idx[2]=[5]\n",
    "special_affilate_idx[4]=[0]\n",
    "special_affilate_idx[22]=[51]\n",
    "special_affilate_idx[32]=[8]\n",
    "special_affilate_idx[54]=[6,11,14]\n",
    "special_pass_idx=[0,5,6,8,11,14,51]\n",
    "padding_O=[0]*len(new_id2labels)\n",
    "padding_O[0]=1\n",
    "padding_category=[0]*len(new_id2labels)\n",
    "padding_category[5]=1\n",
    "padding_SEP=[0]*len(new_id2labels)\n",
    "padding_SEP[6]=1\n",
    "padding_CLS=[0]*len(new_id2labels)\n",
    "padding_CLS[7]=1\n",
    "padding_PAD=[0]*len(new_id2labels)\n",
    "padding_PAD[8]=1 #for pointer ner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "code_folding": [
     0,
     23,
     26
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading...\n",
      "loading...\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists(\"./middle_data/train_postag.pk\"):\n",
    "    postag_set=set()\n",
    "    jieba.enable_paddle() \n",
    "    jieba.enable_parallel(8)\n",
    "    train_cut_words=[]\n",
    "    train_cut_tags=[]\n",
    "    for idx in tqdm(range(len(text_data))):\n",
    "        words = pseg.lcut(text_data[idx]['text'],use_paddle=True) #jieba默认模式\n",
    "        new_words=[w for w,t in words]\n",
    "        new_tags=[t for w,t in words]\n",
    "        postag_set.update(new_tags)\n",
    "        train_cut_words.append([idx,new_words])\n",
    "        train_cut_tags.append([idx,new_tags])\n",
    "    train_cut_words=[e[1] for e in sorted(train_cut_words,key=lambda x:x[0])]\n",
    "    train_cut_tags=[e[1] for e in sorted(train_cut_tags,key=lambda x:x[0])]\n",
    "    id2pos=list(postag_set)\n",
    "    pos2id=dict([(pos,idx) for idx,pos in enumerate(id2pos)])\n",
    "    jieba.enable_parallel(16)\n",
    "    for idx in tqdm(range(len(text_data))):\n",
    "        words = pseg.lcut(text_data[idx]['text'],use_paddle=True) #jieba默认模式\n",
    "        new_words=[w for w,t in words]\n",
    "        assert new_words==train_cut_words[idx]\n",
    "    pk.dump([id2pos,pos2id,train_cut_words,train_cut_tags],open(\"./middle_data/train_postag.pk\",\"wb\"))\n",
    "else:\n",
    "    print(\"loading...\")\n",
    "    id2pos,pos2id,train_cut_words,train_cut_tags=pk.load(open(\"./middle_data/train_postag.pk\",\"rb\"))\n",
    "if not os.path.exists(\"./middle_data/train_postag_BIO.pk\"):\n",
    "    id2pos_BIO=['B-'+e for e in id2pos]+['I-'+e for e in id2pos]\n",
    "    id2pos_BIO.extend(['[CLS]','[SEP]','[PAD]'])\n",
    "    pos2id_BIO=dict([(pos,idx) for idx,pos in enumerate(id2pos_BIO)])\n",
    "\n",
    "    text_data_postag=[]\n",
    "    for idx in tqdm(range(len(text_data))):\n",
    "        assert len(text_data[idx]['text'].strip())==len(\"\".join(train_cut_words[idx]))\n",
    "        indexs=[]\n",
    "        cur_length=0\n",
    "        for e in train_cut_words[idx]:\n",
    "            indexs.append([cur_length,cur_length+len(e)-1])\n",
    "            cur_length+=len(e)\n",
    "        pos_label=np.zeros(len(text_data[idx]['text'])).astype(np.int8)\n",
    "        for i,(b,e) in enumerate(indexs):\n",
    "            assert (text_data[idx]['text'].strip()[b]==train_cut_words[idx][i][0] or _is_whitespace(text_data[idx]['text'].strip()[b])                   or _is_whitespace(train_cut_words[idx][i][0]))                     and (text_data[idx]['text'].strip()[e]==train_cut_words[idx][i][-1]                          or _is_whitespace(text_data[idx]['text'].strip()[e])  or _is_whitespace(train_cut_words[idx][i][-1]))\n",
    "            pos_label[b+1:e+1]=pos2id_BIO['I-'+train_cut_tags[idx][i]]\n",
    "            pos_label[b]=pos2id_BIO['B-'+train_cut_tags[idx][i]]\n",
    "        text_data_postag.append(pos_label)\n",
    "    pk.dump([id2pos_BIO,pos2id_BIO,text_data_postag],open(\"./middle_data/train_postag_BIO.pk\",\"wb\"))\n",
    "else:\n",
    "    print(\"loading...\")\n",
    "    id2pos_BIO,pos2id_BIO,text_data_postag=pk.load(open(\"./middle_data/train_postag_BIO.pk\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading...\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists(\"./middle_data/texts_tfidf_svd.pk\"):\n",
    "    texts_tfidf_svd,tfidf_vectorizer,svd_vectorizer=build_tfidf_svd_matrix(text_data,16)\n",
    "    joblib.dump(tfidf_vectorizer,\"./middle_data/tfidf_vectorizer.sk.model\")\n",
    "    joblib.dump(svd_vectorizer,\"./middle_data/svd_vectorizer.sk.model\")\n",
    "    pk.dump(texts_tfidf_svd,open(\"./middle_data/texts_tfidf_svd.pk\",\"wb\"))\n",
    "else:\n",
    "    print(\"loading...\")\n",
    "    texts_tfidf_svd=pk.load(open(\"./middle_data/texts_tfidf_svd.pk\",\"rb\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# if not os.path.exists(\"./middle_data/train_plan_labels_ner.pk\"):\n",
    "#     train_plan_labels=[]\n",
    "#     for e in tqdm(text_data):\n",
    "#         train_plan_labels.append(_convert_example_to_plan_labels(e,tokenizer))\n",
    "#     for idx in tqdm(range(len(train_plan_labels))):\n",
    "#         for jdx in range(len(train_plan_labels[idx])):\n",
    "#             if sum(train_plan_labels[idx][jdx])==0:\n",
    "#                 train_plan_labels[idx][jdx][0]=1\n",
    "#     pk.dump(train_plan_labels,open(\"./middle_data/train_plan_labels_ner.pk\",\"wb\"))\n",
    "# else:\n",
    "#     print(\"loading...\")\n",
    "#     train_plan_labels=pk.load(open(\"./middle_data/train_plan_labels_ner.pk\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/ipykernel_launcher.py:7: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  import sys\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d0f2d9d32f4b466d9024a7b5ec3edf40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# if not os.path.exists(\"./middle_data/doc_token_info_ner.pk\"):\n",
    "#     ner_doc_tokens=[]\n",
    "#     ner_doc_token_labels=[]\n",
    "#     ner_doc_rels=[]\n",
    "#     ner_token2doc=[]\n",
    "#     ner_doc2doc=[]\n",
    "#     for idx,example in tqdm(enumerate(text_data)):\n",
    "#         tok_to_orig_start_index,tok_to_orig_end_index,orig_to_tok_index,tokens,labels_list,rel_labels=_convert_example_to_record(example,tokenizer)\n",
    "#         rel_list=[e['predicate'] for e in example['spo_list']]\n",
    "# #         if len(set(rel_list))!=len(rel_list):\n",
    "# #                  print(idx)\n",
    "#         for i in range(len(rel_labels)):\n",
    "#             ner_doc_tokens.append(tokens)\n",
    "#             ner_doc_rels.append(rel_labels[i])\n",
    "#             ner_doc_token_labels.append(labels_list[rel_labels[i]])\n",
    "#             ner_token2doc.append([tok_to_orig_start_index,tok_to_orig_end_index,orig_to_tok_index])\n",
    "#             ner_doc2doc.append(idx)\n",
    "#     pk.dump([ner_doc2doc,ner_doc_rels,ner_doc_tokens,ner_doc_token_labels,ner_token2doc],open(\"./middle_data/doc_token_info_ner.pk\",\"wb\"))\n",
    "# else:\n",
    "#     print(\"loading doc..\")\n",
    "#     ner_doc2doc,ner_doc_rels,ner_doc_tokens,ner_doc_token_labels,ner_token2doc=pk.load(open(\"./middle_data/doc_token_info_ner.pk\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/ipykernel_launcher.py:3: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0d04a8e3e85451bba6a1d4d77ec7c88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=191967), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/ipykernel_launcher.py:5: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  \"\"\"\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a16836b0cab4d40acba3dac456012d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=191967), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# if not os.path.exists(\"./middle_data/train_plan_labels_pointer.pk\"):\n",
    "#     train_plan_labels=[]\n",
    "#     for e in tqdm(text_data):\n",
    "#         train_plan_labels.append(_convert_example_to_plan_labels_new(e,tokenizer))\n",
    "#     for idx in tqdm(range(len(train_plan_labels))):\n",
    "#         for jdx in range(len(train_plan_labels[idx])):\n",
    "#             if sum(train_plan_labels[idx][jdx])==0:\n",
    "#                 train_plan_labels[idx][jdx][0]=1\n",
    "#     pk.dump(train_plan_labels,open(\"./middle_data/train_plan_labels_pointer.pk\",\"wb\"))\n",
    "# else:\n",
    "#     print(\"loading...\")\n",
    "#     train_plan_labels=pk.load(open(\"./middle_data/train_plan_labels_pointer.pk\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/ipykernel_launcher.py:7: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  import sys\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89eb943fefb34b19b6f6607706d71ef4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# if not os.path.exists(\"./middle_data/doc_token_info_pointer.pk\"):\n",
    "#     ner_doc_tokens=[]\n",
    "#     ner_doc_token_labels  \n",
    "#     ner_doc_rels=[]\n",
    "#     ner_token2doc=[]\n",
    "#     ner_doc2doc=[]\n",
    "#     for idx,example in tqdm(enumerate(text_data)):\n",
    "#         tok_to_orig_start_index,tok_to_orig_end_index,orig_to_tok_index,tokens,labels_list,rel_labels=_convert_example_to_record_new(example,tokenizer)\n",
    "#         rel_list=[e['predicate'] for e in example['spo_list']]\n",
    "# #         if len(set(rel_list))!=len(rel_list):\n",
    "# #                  print(idx)\n",
    "#         for i in range(len(rel_labels)):\n",
    "#             ner_doc_tokens.append(tokens)\n",
    "#             ner_doc_rels.append(rel_labels[i])\n",
    "#             ner_doc_token_labels.append(labels_list[rel_labels[i]])\n",
    "#             ner_token2doc.append([tok_to_orig_start_index,tok_to_orig_end_index,orig_to_tok_index])\n",
    "#             ner_doc2doc.append(idx)\n",
    "#     for idx in tqdm(range(len(ner_doc_token_labels))):\n",
    "#         for jdx in range(len(ner_doc_token_labels[idx])):\n",
    "#             if sum(ner_doc_token_labels[idx][jdx])==0:\n",
    "#                 ner_doc_token_labels[idx][jdx][0]=1\n",
    "#     pk.dump([ner_doc2doc,ner_doc_rels,ner_doc_tokens,ner_doc_token_labels,ner_token2doc],open(\"./middle_data/doc_token_info_pointer.pk\",\"wb\"))\n",
    "# else:\n",
    "#     print(\"loading doc..\")\n",
    "#     ner_doc2doc,ner_doc_rels,ner_doc_tokens,ner_doc_token_labels,ner_token2doc=pk.load(open(\"./middle_data/doc_token_info_pointer.pk\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "code_folding": [
     1
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading example..\n"
     ]
    }
   ],
   "source": [
    "\n",
    "if not os.path.exists(\"./middle_data/ner_examples_pointer.pk\"):\n",
    "    ner_examples=[]\n",
    "    example2doc=[]\n",
    "    doc2example=[[] for i in range(len(ner_doc_tokens))]\n",
    "    for idx,(tokens,rel,labels,token2doc) in tqdm(enumerate(zip(ner_doc_tokens,ner_doc_rels,ner_doc_token_labels,ner_token2doc))):\n",
    "        tfidf_svd=texts_tfidf_svd[ner_doc2doc[idx]]\n",
    "        postag=text_data_postag[ner_doc2doc[idx]]\n",
    "        plan_labels=train_plan_labels[ner_doc2doc[idx]]\n",
    "        index=[0,new_kglabels2id['B-'+id2rels[rel]+\"-OB\"],new_kglabels2id['E-'+id2rels[rel]+\"-OB\"],\\\n",
    "           new_kglabels2id['B-'+id2rels[rel]+\"-SUB\"],new_kglabels2id['E-'+id2rels[rel]+\"-SUB\"]]\n",
    "        plan_labels=np.concatenate([np.array(plan_labels)[:,index],np.zeros((len(plan_labels),4)).astype(np.int64)],axis=1).tolist()\n",
    "        cur_idx=0\n",
    "        rel_text=id2rel_text[rel]\n",
    "        rel_postag=rel_data_postag[rel]\n",
    "        rel_token2doc=id2rel_token2text[rel]\n",
    "        while len(tokens)-cur_idx> (max_len-3-len(rel_text)):\n",
    "            temp_tokens=tokens[cur_idx:cur_idx+max_len-3-len(rel_text)]\n",
    "            temp_labels=labels[cur_idx:cur_idx+max_len-3-len(rel_text)]\n",
    "            temp_plan_labels=plan_labels[cur_idx:cur_idx+max_len-3-len(rel_text)]\n",
    "            temp_token2doc=[e[cur_idx:cur_idx+max_len-3-len(rel_text)] if ind<2 else e for ind,e in enumerate(token2doc)]\n",
    "            #训练时把这些label里面妹有 3 4 1 2 的样本去掉\n",
    "            compare_labels=np.array(temp_labels).argmax(-1)\n",
    "            if (1 in  compare_labels) or (2 in  compare_labels) or (3 in  compare_labels) or (4 in  compare_labels):\n",
    "                example2doc.append(idx)\n",
    "                ner_examples.append(create_example_new(temp_tokens,rel,temp_labels,temp_token2doc,tokenizer,rel_text,tfidf_svd,postag,rel_postag,rel_token2doc,temp_plan_labels))\n",
    "                doc2example[idx].append(len(ner_examples)-1)\n",
    "            cur_idx+=max_len-3-len(rel_text)\n",
    "        if cur_idx<len(tokens):\n",
    "            temp_tokens=tokens[cur_idx:]\n",
    "            temp_labels=labels[cur_idx:]\n",
    "            temp_plan_labels=plan_labels[cur_idx:]\n",
    "            temp_token2doc=[e[cur_idx:cur_idx+max_len-3-len(rel_text)] if ind<2 else e for ind,e in enumerate(token2doc)]\n",
    "            compare_labels=np.array(temp_labels).argmax(-1)\n",
    "            if (1 in  compare_labels) or (2 in  compare_labels) or (3 in  compare_labels) or (4 in  compare_labels):\n",
    "                example2doc.append(idx)\n",
    "                ner_examples.append(create_example_new(temp_tokens,rel,temp_labels,temp_token2doc,tokenizer,rel_text,tfidf_svd,postag,rel_postag,rel_token2doc,temp_plan_labels))    \n",
    "                doc2example[idx].append(len(ner_examples)-1)\n",
    "\n",
    "    pk.dump([doc2example,example2doc,ner_examples],open(\"./middle_data/ner_examples_pointer.pk\",\"wb\"))\n",
    "else:\n",
    "    print(\"loading example..\")\n",
    "    doc2example,example2doc,ner_examples=pk.load(open(\"./middle_data/ner_debug_examples_pointer.pk\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pk.dump([doc2example,example2doc,ner_examples[:10000]+ner_examples[-1000:]],open(\"./middle_data/ner_debug_examples_pointer.pk\",\"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ner_train_dataset=NerDataset(ner_examples[:10000])\n",
    "ner_val_dataset=NerDataset(ner_examples[-1000:])\n",
    "# ner_train_dataset=NerDataset(ner_examples[:268432])\n",
    "# ner_val_dataset=NerDataset(ner_examples[268432:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_dropout_prob = 0.1\n",
    "num_rel_labels = len(id2rels)\n",
    "num_token_labels=len(id2labels)\n",
    "learning_rate = 2e-5\n",
    "weight_decay = 0\n",
    "epochs = 3\n",
    "batch_size = 16\n",
    "adam_epsilon=1e-8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# idx=10\n",
    "# for t,l,pl in zip(tokenizer.convert_ids_to_tokens(ner_examples[idx]['input_ids']),\\\n",
    "#                   ner_examples[idx]['labels'][:,0:5],ner_examples[idx]['plan_labels'][:,0:5]):\n",
    "#     print(t,new_id2labels[l.numpy().argmax()],new_id2labels[pl.numpy().argmax()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# epoch_iterator=Data.DataLoader(ner_train_dataset,batch_size=2)\n",
    "# for step, batch in enumerate(epoch_iterator):\n",
    "#         model.eval()\n",
    "#         with torch.no_grad():\n",
    "#             inputs = {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3],\"label_tokens\":batch[4],\"plan_label\":batch[7].float()}\n",
    "#             inputs[\"token_type_ids\"]=batch[2]\n",
    "#             outputs = model(**inputs)\n",
    "#         break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "code_folding": [
     0,
     19,
     93,
     101,
     155,
     161,
     325,
     420,
     441,
     517,
     681
    ]
   },
   "outputs": [],
   "source": [
    "class GCNN_block(nn.Module):\n",
    "    def __init__(self,input_channel,output_channel,kernel_size,padding,dilation=1):\n",
    "        super(GCNN_block,self).__init__()\n",
    "        self.input_channel=input_channel\n",
    "        self.output_channel=output_channel\n",
    "        self.conv1=nn.Conv1d(input_channel,output_channel,kernel_size,padding=padding,dilation=dilation)\n",
    "        self.conv2=nn.Conv1d(input_channel,output_channel,kernel_size,padding=padding,dilation=dilation)\n",
    "        if input_channel !=output_channel:\n",
    "            self.trans=nn.Conv1d(input_channel,output_channel,1)\n",
    "    def forward(self,args):\n",
    "        X,attention_mask=args[0],args[1]\n",
    "        X=X*attention_mask.unsqueeze(1).float()\n",
    "        gate=torch.sigmoid(self.conv2(X))\n",
    "        if self.input_channel==self.output_channel:\n",
    "            Y=X*(1-gate)+self.conv1(X)*gate\n",
    "        else:\n",
    "            Y=self.trans(X)*(1-gate)+self.conv1(X)*gate\n",
    "        Y=Y*attention_mask.unsqueeze(1).float()\n",
    "        return Y,attention_mask\n",
    "class BERT_BiLSTM_CRF(nn.Module):\n",
    "    def __init__(self,bert_dir,config_file,num_rel_labels,num_token_labels,hidden_dropout_prob,need_birnn=True,need_cnn=False,rnn_dim=128):\n",
    "        super(BERT_BiLSTM_CRF, self).__init__()\n",
    "        self.num_rel_labels = num_rel_labels\n",
    "        self.num_token_labels= num_token_labels\n",
    "        self.bert =BertModel.from_pretrained(bert_dir,config=os.path.join(bert_dir,config_file), \\\n",
    "                                             hidden_dropout_prob=hidden_dropout_prob,output_hidden_states=True, output_attentions=True)\n",
    "        self.dropout = nn.Dropout(hidden_dropout_prob)\n",
    "        out_dim =self.bert.pooler.dense.out_features\n",
    "        hidden_size=out_dim\n",
    "        self.rel_classifier = nn.Linear(out_dim, self.num_rel_labels)\n",
    "        self.need_birnn = need_birnn\n",
    "        self.need_cnn=need_cnn\n",
    "        # 如果为False，则不要BiLSTM层\n",
    "        if need_birnn:\n",
    "            self.birnn = nn.LSTM(out_dim, rnn_dim, num_layers=1, bidirectional=True, batch_first=True)\n",
    "            out_dim = rnn_dim*2\n",
    "        if need_cnn:\n",
    "            self.DGCNN=nn.Sequential(GCNN_block(hidden_size,hidden_size,1,padding=0,dilation=1),\n",
    "                       GCNN_block(hidden_size,hidden_size,3,padding=1,dilation=1),\n",
    "                       GCNN_block(hidden_size,hidden_size,3,padding=2,dilation=2),\n",
    "                       GCNN_block(hidden_size,hidden_size,3,padding=4,dilation=4))\n",
    "            out_dim=hidden_size\n",
    "            #attention/batch norm+relu/ pooling/什么都不用\n",
    "        \n",
    "        self.hidden2tag = nn.Linear(out_dim, self.num_token_labels)\n",
    "        self.crf = CRF(self.num_token_labels, batch_first=True)\n",
    "    def forward(\n",
    "        self,\n",
    "        input_ids=None,\n",
    "        attention_mask=None,\n",
    "        token_type_ids=None,\n",
    "        labels=None,\n",
    "        label_tokens=None\n",
    "    ):\n",
    "\n",
    "        outputs = self.bert(\n",
    "            input_ids,\n",
    "            attention_mask=attention_mask,\n",
    "            token_type_ids=token_type_ids)\n",
    "\n",
    "        seq_output,pooled_output = outputs[0],outputs[1]\n",
    "        pooled_output = self.dropout(pooled_output)\n",
    "        emissions=self.tag_outputs(seq_output,attention_mask)\n",
    "        rel_logits = self.rel_classifier(pooled_output)\n",
    "        seq_logits = emissions\n",
    "        outputs = (rel_logits,seq_logits) + outputs[2:]  # add hidden states and attention if they are here\n",
    "        loss_full=None\n",
    "        if labels is not None:\n",
    "            loss_rel = nn.BCEWithLogitsLoss(reduction=\"none\")\n",
    "            l_r = ((loss_rel(rel_logits.view(-1, self.num_rel_labels), labels.view(-1,self.num_rel_labels).float())).sum(dim=-1)).mean()\n",
    "            loss_full=l_r\n",
    "\n",
    "        if label_tokens is not None:\n",
    "            l_t = -1*self.crf(emissions, label_tokens, mask=attention_mask.byte(),reduction='mean')\n",
    "            loss_full+=l_t\n",
    "        if loss_full is not None:\n",
    "            outputs = (loss_full,) + outputs\n",
    "        return outputs  # (loss), logits, (hidden_states), (attentions)\n",
    "\n",
    "    def tag_outputs(self, sequence_output,attention_mask):\n",
    "        if self.need_birnn:\n",
    "            sequence_output, _ = self.birnn(sequence_output)\n",
    "        if self.need_cnn:\n",
    "            sequence_output=self.DGCNN([sequence_output.permute(0,2,1),attention_mask])[0].permute(0,2,1)\n",
    "        sequence_output = self.dropout(sequence_output)\n",
    "        emissions = self.hidden2tag(sequence_output)\n",
    "        return emissions\n",
    "    \n",
    "    def predict(self, input_ids,attention_mask=None,token_type_ids=None):\n",
    "        outputs = self.bert(input_ids, attention_mask=attention_mask,token_type_ids=token_type_ids )\n",
    "        sequence_output = outputs[0]\n",
    "        emissions = self.tag_outputs(sequence_output,attention_mask)\n",
    "        return self.crf.decode(emissions, attention_mask.byte())\n",
    "class MaskedSoftmaxCELoss(nn.CrossEntropyLoss):\n",
    "    def __init__(self):\n",
    "            super(MaskedSoftmaxCELoss,self).__init__(reduction='none')\n",
    "    def forward(self,pred,label,mask=None):\n",
    "            loss=super(MaskedSoftmaxCELoss,self).forward((pred).contiguous().view(-1,pred.shape[-1]),(label).contiguous().view(-1).long()).view(label.shape)\n",
    "            if mask is not None:\n",
    "                loss*=mask.float().contiguous()\n",
    "            return loss\n",
    "class BertSeqLabeling(nn.Module):\n",
    "    def __init__(self,bert_dir,config_file,num_rel_labels,num_token_labels,hidden_dropout_prob):\n",
    "        super(BertSeqLabeling,self).__init__()\n",
    "        self.num_rel_labels = num_rel_labels\n",
    "        self.num_token_labels= num_token_labels\n",
    "        self.bert =BertModel.from_pretrained(bert_dir,config=os.path.join(bert_dir,config_file),                                              hidden_dropout_prob=hidden_dropout_prob,output_hidden_states=True, output_attentions=True)\n",
    "        self.dropout = nn.Dropout(hidden_dropout_prob)\n",
    "        self.rel_classifier = nn.Linear(self.bert.pooler.dense.out_features, self.num_rel_labels)\n",
    "        self.seq_classifier=nn.Linear(self.bert.pooler.dense.out_features,self.num_token_labels)\n",
    "    def forward(\n",
    "        self,\n",
    "        input_ids=None,\n",
    "        attention_mask=None,\n",
    "        token_type_ids=None,\n",
    "        position_ids=None,\n",
    "        head_mask=None,\n",
    "        inputs_embeds=None,\n",
    "        labels=None,\n",
    "        label_tokens=None,\n",
    "    ):\n",
    "\n",
    "        outputs = self.bert(\n",
    "            input_ids,\n",
    "            attention_mask=attention_mask,\n",
    "            token_type_ids=token_type_ids,\n",
    "            position_ids=position_ids,\n",
    "            head_mask=head_mask,\n",
    "            inputs_embeds=inputs_embeds,\n",
    "        )\n",
    "\n",
    "        seq_output,pooled_output = outputs[0],outputs[1]\n",
    "        pooled_output = self.dropout(pooled_output)\n",
    "        seq_output=self.dropout(seq_output)\n",
    "        rel_logits = self.rel_classifier(pooled_output)\n",
    "        seq_logits = self.seq_classifier(seq_output)\n",
    "        outputs = (rel_logits,seq_logits) + outputs[2:]  # add hidden states and attention if they are here\n",
    "        loss_full=None\n",
    "        if labels is not None:\n",
    "            loss_rel = nn.BCEWithLogitsLoss(reduction=\"none\")\n",
    "            l_r = ((loss_rel(rel_logits.view(-1, self.num_rel_labels), labels.view(-1,self.num_rel_labels).float())).sum(dim=-1)).mean()\n",
    "            loss_full=l_r\n",
    "\n",
    "\n",
    "        if label_tokens is not None:\n",
    "            loss_token =MaskedSoftmaxCELoss()\n",
    "            # Only keep active parts of the loss\n",
    "            if attention_mask is not None:\n",
    "                l_t = (loss_token(seq_logits, label_tokens,attention_mask)).sum(dim=-1).mean()\n",
    "            else:\n",
    "                l_t = loss_token(seq_logits, label_tokens).sum(dim=-1).mean()\n",
    "            loss_full+=l_t\n",
    "        if loss_full is not None:\n",
    "            outputs = (loss_full,) + outputs\n",
    "        return outputs  # (loss), logits, (hidden_states), (attentions)\n",
    "def set_seed(args):\n",
    "    random.seed(args.seed)\n",
    "    np.random.seed(args.seed)\n",
    "    torch.manual_seed(args.seed)\n",
    "    if args.n_gpu > 0:\n",
    "        torch.cuda.manual_seed_all(args.seed)\n",
    "def train(args, train_dataset,val_dataset, model, tokenizer):\n",
    "#     \"\"\" Train the model \"\"\"\n",
    "#     if args.local_rank in [-1, 0]:\n",
    "#         tb_writer = SummaryWriter()\n",
    "\n",
    "#     args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)\n",
    "    train_sampler = RandomSampler(train_dataset) # if args.local_rank == -1 else DistributedSampler(train_dataset)\n",
    "    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)\n",
    "\n",
    "    if args.max_steps > 0:\n",
    "        t_total = args.max_steps\n",
    "        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1\n",
    "    else:\n",
    "        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs\n",
    "\n",
    "    # Prepare optimizer and schedule (linear warmup and decay)\n",
    "    no_decay = [\"bias\", \"LayerNorm.weight\"]\n",
    "    optimizer_grouped_parameters = [\n",
    "        {\n",
    "            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
    "            \"weight_decay\": args.weight_decay,\n",
    "        },\n",
    "        {\"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], \"weight_decay\": 0.0},\n",
    "    ]\n",
    "\n",
    "    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)\n",
    "    scheduler = get_linear_schedule_with_warmup(\n",
    "        optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total\n",
    "    )\n",
    "\n",
    "    # Check if saved optimizer or scheduler states exist\n",
    "    if os.path.isfile(os.path.join(args.model_name_or_path, \"optimizer.pt\")) and os.path.isfile(\n",
    "        os.path.join(args.model_name_or_path, \"scheduler.pt\")\n",
    "    ):\n",
    "        logger.info(\"  loading optimizer and scheduler...\")\n",
    "        # Load in optimizer and scheduler states\n",
    "        optimizer.load_state_dict(torch.load(os.path.join(args.model_name_or_path, \"optimizer.pt\")))\n",
    "#         scheduler.load_state_dict(torch.load(os.path.join(args.model_name_or_path, \"scheduler.pt\")))\n",
    "    else:\n",
    "        logger.info(\"  No optimizer and scheduler,we build a new one\")        \n",
    "\n",
    "    if args.fp16:\n",
    "        try:\n",
    "            from apex import amp\n",
    "        except ImportError:\n",
    "            raise ImportError(\"Please install apex from https://www.github.com/nvidia/apex to use fp16 training.\")\n",
    "        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)\n",
    "\n",
    "    # multi-gpu training (should be after apex fp16 initialization)\n",
    "    if args.n_gpu > 1:\n",
    "        model = torch.nn.DataParallel(model,device_ids=args.card_list)\n",
    "\n",
    "\n",
    "    # Train!\n",
    "    logger.info(\"***** Running training *****\")\n",
    "    logger.info(\"  Num examples = %d\", len(train_dataset))\n",
    "    logger.info(\"  Num Epochs = %d\", args.num_train_epochs)\n",
    "#     logger.info(\"  Instantaneous batch size per GPU = %d\", args.per_gpu_train_batch_size)\n",
    "    logger.info(\n",
    "        \"  Total train batch size (w. parallel, distributed & accumulation) = %d\",\n",
    "        args.train_batch_size\n",
    "        * args.gradient_accumulation_steps\n",
    "#         * (torch.distributed.get_world_size() if args.local_rank != -1 else 1),\n",
    "    )\n",
    "    logger.info(\"  Gradient Accumulation steps = %d\", args.gradient_accumulation_steps)\n",
    "    logger.info(\"  Total optimization steps = %d\", t_total)\n",
    "\n",
    "    global_step = 0\n",
    "    epochs_trained = 0\n",
    "    steps_trained_in_current_epoch = 0\n",
    "    # Check if continuing training from a checkpoint\n",
    "    if os.path.exists(args.model_name_or_path):\n",
    "        # set global_step to global_step of last saved checkpoint from model path\n",
    "        try:\n",
    "            global_step = int(args.model_name_or_path.split(\"-\")[-1].split(\"/\")[0])\n",
    "        except ValueError:\n",
    "            global_step = 0\n",
    "        epochs_trained = global_step // (len(train_dataloader) // args.gradient_accumulation_steps)\n",
    "        steps_trained_in_current_epoch = global_step % (len(train_dataloader) // args.gradient_accumulation_steps)\n",
    "\n",
    "        logger.info(\"  Continuing training from checkpoint, will skip to saved global_step\")\n",
    "        logger.info(\"  Continuing training from epoch %d\", epochs_trained)\n",
    "        logger.info(\"  Continuing training from global step %d\", global_step)\n",
    "        logger.info(\"  Will skip the first %d steps in the first epoch\", steps_trained_in_current_epoch)\n",
    "\n",
    "    tr_loss, logging_loss = 0.0, 0.0\n",
    "    model.zero_grad()\n",
    "    train_iterator = tqdm(range(\n",
    "        epochs_trained, int(args.num_train_epochs)), desc=\"Epoch\")\n",
    "    set_seed(args)  # Added here for reproductibility\n",
    "    for _ in train_iterator:\n",
    "        epoch_iterator = tqdm(train_dataloader, desc=\"Iteration\")\n",
    "        for step, batch in enumerate(epoch_iterator):\n",
    "\n",
    "            # Skip past any already trained steps if resuming training\n",
    "            if steps_trained_in_current_epoch > 0:\n",
    "                if  (step + 1) % args.gradient_accumulation_steps == 0: \n",
    "                        steps_trained_in_current_epoch -= 1\n",
    "                continue\n",
    "\n",
    "            model.train()\n",
    "            batch = tuple(t.to(args.device) for t in batch[:-1])\n",
    "            inputs = {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3],\"label_tokens\":batch[4]}\n",
    "            inputs[\"token_type_ids\"]=batch[2]\n",
    "#             if args.model_type != \"distilbert\":\n",
    "#                 inputs[\"token_type_ids\"] = (\n",
    "#                     batch[2] if args.model_type in [\"bert\", \"xlnet\", \"albert\"] else None\n",
    "#                 )  # XLM, DistilBERT, RoBERTa, and XLM-RoBERTa don't use segment_ids\n",
    "            outputs = model(**inputs)\n",
    "            loss = outputs[0]  # model outputs are always tuple in transformers (see doc)\n",
    "\n",
    "            if args.n_gpu > 1:\n",
    "                loss = loss.mean()  # mean() to average on multi-gpu parallel training\n",
    "            if args.gradient_accumulation_steps > 1:\n",
    "                loss = loss / args.gradient_accumulation_steps\n",
    "\n",
    "            if args.fp16:\n",
    "                with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "                    scaled_loss.backward()\n",
    "            else:\n",
    "                loss.backward()\n",
    "            logger.info(\"  step:%d loss %.3f\", step,loss.item())\n",
    "\n",
    "            tr_loss += loss.item()\n",
    "            if (step + 1) % args.gradient_accumulation_steps == 0:\n",
    "                if args.fp16:\n",
    "                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)\n",
    "                else:\n",
    "                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)\n",
    "\n",
    "                optimizer.step()\n",
    "                scheduler.step()  # Update learning rate schedule\n",
    "                model.zero_grad()\n",
    "                global_step += 1\n",
    "\n",
    "                if  args.save_steps > 0 and global_step % args.save_steps == 0:\n",
    "                    # Save model checkpoint\n",
    "                    results = evaluate(args, val_dataset,model, tokenizer)\n",
    "                    output_dir = os.path.join(args.output_dir, \"checkpoint-{}\".format(global_step))\n",
    "                    if not os.path.exists(output_dir):\n",
    "                        os.makedirs(output_dir)\n",
    "                    model_to_save = (\n",
    "                        model.module if hasattr(model, \"module\") else model\n",
    "                    )  # Take care of distributed/parallel training\n",
    "                    torch.save(model_to_save.state_dict(),os.path.join(output_dir,\"model.pt\"))\n",
    "                    tokenizer.save_pretrained(output_dir)\n",
    "\n",
    "                    torch.save(args, os.path.join(output_dir, \"training_args.bin\"))\n",
    "                    logger.info(\"Saving model checkpoint to %s\", output_dir)\n",
    "                    if args.fp16:\n",
    "                        torch.save(amp.state_dict(),os.path.join(output_dir, \"amp.pt\"))\n",
    "                    torch.save(optimizer.state_dict(), os.path.join(output_dir, \"optimizer.pt\"))\n",
    "                    torch.save(scheduler.state_dict(), os.path.join(output_dir, \"scheduler.pt\"))\n",
    "                    logger.info(\"Saving optimizer and scheduler states to %s\", output_dir)\n",
    "\n",
    "            if args.max_steps > 0 and global_step > args.max_steps:\n",
    "                epoch_iterator.close()\n",
    "                break\n",
    "        if args.max_steps > 0 and global_step > args.max_steps:\n",
    "            train_iterator.close()\n",
    "            break\n",
    "\n",
    "\n",
    "    return global_step, tr_loss / global_step\n",
    "def evaluate(args, eval_dataset,model, tokenizer, prefix=\"\"):\n",
    "    eval_output_dir = args.output_dir \n",
    "\n",
    "    results = {}\n",
    "\n",
    "    if not os.path.exists(eval_output_dir) :\n",
    "        os.makedirs(eval_output_dir)\n",
    "\n",
    "    eval_sampler = SequentialSampler(eval_dataset)\n",
    "    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size,num_workers=0)\n",
    "\n",
    "    # multi-gpu eval\n",
    "#         if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):\n",
    "#             model = torch.nn.DataParallel(model)\n",
    "\n",
    "    # Eval!\n",
    "    logger.info(\"***** Running evaluation {} *****\".format(prefix))\n",
    "    logger.info(\"  Num examples = %d\", len(eval_dataset))\n",
    "    logger.info(\"  Batch size = %d\", args.eval_batch_size)\n",
    "    eval_loss = 0.0\n",
    "    nb_eval_steps = 0\n",
    "    rel_preds=None\n",
    "    rel_labels=None\n",
    "    token_preds=None\n",
    "    token_labels=None\n",
    "    for batch in tqdm(eval_dataloader, desc=\"Evaluating\"):\n",
    "        model.eval()\n",
    "        batch = tuple(t.to(args.device) for t in batch[:-1])\n",
    "\n",
    "        with torch.no_grad():\n",
    "            inputs = {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3],\"label_tokens\":batch[4]}\n",
    "            inputs[\"token_type_ids\"]=batch[2]\n",
    "            temp_inputs={\"input_ids\": batch[0], \"attention_mask\": batch[1],\"token_type_ids\":batch[2]}\n",
    "#                 if args.model_type != \"distilbert\":\n",
    "#                     inputs[\"token_type_ids\"] = (\n",
    "#                         batch[2] if args.model_type in [\"bert\", \"xlnet\", \"albert\"] else None\n",
    "#                     )  # XLM, DistilBERT, RoBERTa, and XLM-RoBERTa don't use segment_ids\n",
    "            outputs = model(**inputs)\n",
    "            model_to_predict = (\n",
    "                model.module if hasattr(model, \"module\") else model\n",
    "            )  # Take care of distributed/parallel training\n",
    "            token_pred=model_to_predict.predict(**temp_inputs)\n",
    "            tmp_eval_loss, rel_logits, token_logits = outputs[:3]\n",
    "            \n",
    "            eval_loss += tmp_eval_loss.mean().item()\n",
    "        nb_eval_steps += 1\n",
    "        mask=batch[1].view(-1).detach().cpu().numpy()\n",
    "        rel_pred = F.softmax(rel_logits,dim=-1)\n",
    "        rel_pred =torch.argmax(rel_pred,dim=-1).detach().cpu().numpy()\n",
    "        rel_label=torch.argmax(inputs[\"labels\"],dim=-1).detach().cpu().numpy()\n",
    "        if rel_preds is None:\n",
    "            rel_preds=rel_pred\n",
    "        else:\n",
    "            rel_preds=np.append(rel_preds,rel_pred,axis=0)\n",
    "        if rel_labels is None:\n",
    "            rel_labels=rel_label\n",
    "        else:\n",
    "            rel_labels=np.append(rel_labels,rel_label,axis=0)     \n",
    "\n",
    "            \n",
    "        #token classify\n",
    "#         token_pred = F.softmax(token_logits,dim=-1)\n",
    "#         token_pred =torch.argmax(token_pred,dim=-1).view(-1).detach().cpu().numpy()*mask\n",
    "        for i in range(len(token_pred)):\n",
    "            token_pred[i]+=[label2ids['[PAD]']]*(max_len-len(token_pred[i]))\n",
    "        token_pred=np.array(token_pred).reshape(-1)*mask\n",
    "        token_label=inputs[\"label_tokens\"].view(-1).detach().cpu().numpy()*mask\n",
    "        if token_preds is None:\n",
    "            token_preds=token_pred\n",
    "        else:\n",
    "            token_preds=np.append(token_preds,token_pred,axis=0)\n",
    "        if token_labels is None:\n",
    "            token_labels=token_label\n",
    "        else:\n",
    "            token_labels=np.append(token_labels,token_label,axis=0) \n",
    "        \n",
    "\n",
    "\n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    print(\"准召报告：\")\n",
    "    df_precison_recall_f1=pd.DataFrame(metrics.precision_recall_fscore_support(rel_labels,rel_preds),index=['precison','recall','f1','number'])\n",
    "    display(HTML(df_precison_recall_f1.to_html()))\n",
    "    df_precison_recall_f2=pd.DataFrame(metrics.precision_recall_fscore_support(token_labels,token_preds),index=['precison','recall','f1','number'])\n",
    "    display(HTML(df_precison_recall_f2.to_html()))\n",
    "    result = {\"cls\":    df_precison_recall_f1,'ner':    df_precison_recall_f2}\n",
    "    results.update(result)\n",
    "\n",
    "    output_eval_file = os.path.join(eval_output_dir, prefix, \"eval_results.txt\")\n",
    "    with open(output_eval_file, \"a\") as writer:\n",
    "        logger.info(\"***** Eval results {} *****\".format(prefix))\n",
    "        for key in sorted(result.keys()):\n",
    "            logger.info(\"  %s = %s\", key, str(result[key]))\n",
    "            writer.write(\"%s = %s\\n\" % (key, str(result[key])))\n",
    "\n",
    "    return rel_labels,rel_preds,token_preds,token_labels\n",
    "class Conditional_LayerNorm(nn.Module):\n",
    "    def __init__(self,features,conditional_dim,eps=1e-6):\n",
    "        super(Conditional_LayerNorm,self).__init__()\n",
    "        self.gamma=nn.Parameter(torch.ones(features))\n",
    "        self.beta=nn.Parameter(torch.zeros(features))\n",
    "        self.trans_gamma=nn.Linear(conditional_dim,self.gamma.shape[-1])\n",
    "        self.trans_beta=nn.Linear(conditional_dim,self.beta.shape[-1])\n",
    "        torch.nn.init.constant_(self.trans_gamma.weight,val=0)\n",
    "        torch.nn.init.constant_(self.trans_gamma.bias,val=0)\n",
    "        torch.nn.init.constant_(self.trans_beta.weight,val=0)\n",
    "        torch.nn.init.constant_(self.trans_beta.bias,val=0)\n",
    "        self.eps=eps\n",
    "    def forward(self,X,condition):\n",
    "        mean=X.mean(-1,keepdim=True)\n",
    "        std=X.std(-1,keepdim=True)\n",
    "        cond_gamma=self.trans_gamma(condition)\n",
    "        cond_beta=self.trans_beta(condition)\n",
    "        if condition.dim()<X.dim(): #condition是固定维度\n",
    "            return (self.gamma+cond_gamma).unsqueeze(1)*(X-mean)/(std+self.eps)+(self.beta+cond_beta).unsqueeze(1)\n",
    "        else:#condition是sequence\n",
    "            return (self.gamma+cond_gamma)*(X-mean)/(std+self.eps)+(self.beta+cond_beta)\n",
    "class BERT_BiLSTM_POINTER(nn.Module):\n",
    "    def __init__(self,bert_dir,config_file,num_rel_labels,num_token_labels,hidden_dropout_prob,\\\n",
    "                 need_birnn=True,need_cnn=False,rnn_dim=128,use_plan=False,p=1):\n",
    "        super(BERT_BiLSTM_POINTER, self).__init__()\n",
    "        self.num_rel_labels = num_rel_labels\n",
    "        self.num_token_labels= 4\n",
    "        self.bert =BertModel.from_pretrained(bert_dir,config=os.path.join(bert_dir,config_file), \\\n",
    "                                             hidden_dropout_prob=hidden_dropout_prob,output_hidden_states=True, output_attentions=True)\n",
    "        self.dropout = nn.Dropout(hidden_dropout_prob)\n",
    "        out_dim =self.bert.pooler.dense.out_features\n",
    "        hidden_size=out_dim\n",
    "        self.rel_classifier = nn.Linear(out_dim, self.num_rel_labels)\n",
    "        self.need_birnn = need_birnn\n",
    "        self.need_cnn=need_cnn\n",
    "        self.p=p\n",
    "        self.use_plan=use_plan\n",
    "        if use_plan:\n",
    "            self.cond_ln=Conditional_LayerNorm(hidden_size,9)\n",
    "        if need_birnn:\n",
    "            self.birnn = nn.LSTM(out_dim, rnn_dim, num_layers=1, bidirectional=True, batch_first=True)\n",
    "            out_dim = rnn_dim*2\n",
    "        if need_cnn:\n",
    "            self.DGCNN=nn.Sequential(GCNN_block(hidden_size,hidden_size,1,padding=0,dilation=1),\n",
    "                       GCNN_block(hidden_size,hidden_size,3,padding=1,dilation=1),\n",
    "                       GCNN_block(hidden_size,hidden_size,3,padding=2,dilation=2),\n",
    "                       GCNN_block(hidden_size,hidden_size,3,padding=4,dilation=4))\n",
    "            out_dim=hidden_size\n",
    "        self.hidden2tag = nn.Linear(out_dim, self.num_token_labels)\n",
    "    def forward(\n",
    "        self,\n",
    "        input_ids=None,\n",
    "        attention_mask=None,\n",
    "        token_type_ids=None,\n",
    "        labels=None,\n",
    "        label_tokens=None,\n",
    "        plan_label=None\n",
    "    ):\n",
    "        label_tokens=(label_tokens[:,:,1:5]!=0).long()\n",
    "        outputs = self.bert(\n",
    "            input_ids,\n",
    "            attention_mask=attention_mask,\n",
    "            token_type_ids=token_type_ids)\n",
    "\n",
    "        seq_output,pooled_output = outputs[0],outputs[1]\n",
    "        pooled_output = self.dropout(pooled_output)\n",
    "        if self.use_plan:\n",
    "            plan_output=F.adaptive_avg_pool1d(plan_label.permute(0,2,1),1).squeeze(-1)\n",
    "            seq_output=self.cond_ln(seq_output,plan_output)\n",
    "        emissions=self.tag_outputs(seq_output,attention_mask)\n",
    "        rel_logits = self.rel_classifier(pooled_output)\n",
    "        seq_logits = emissions\n",
    "        outputs = (rel_logits,seq_logits) + outputs[2:]  # add hidden states and attention if they are here\n",
    "        loss_full=None\n",
    "        if labels is not None:\n",
    "            loss_rel = nn.BCEWithLogitsLoss(reduction=\"none\")\n",
    "            l_r = ((loss_rel(rel_logits.view(-1, self.num_rel_labels), labels.view(-1,self.num_rel_labels).float())).sum(dim=-1)).mean()\n",
    "            loss_full=l_r\n",
    "\n",
    "        if label_tokens is not None:\n",
    "            #(batch_size,seq_len,num_token_labels)\n",
    "            loss_token=nn.BCELoss(reduction='none')\n",
    "            l_t=(torch.sum((loss_token(torch.sigmoid(seq_logits)**self.p, label_tokens.float()))\\\n",
    "                 *attention_mask.float().unsqueeze(-1)))/input_ids.shape[0]\n",
    "            loss_full+=l_t\n",
    "        if loss_full is not None:\n",
    "            outputs = (loss_full,) + outputs\n",
    "        return outputs  # (loss), logits, (hidden_states), (attentions)\n",
    "\n",
    "    def tag_outputs(self, sequence_output,attention_mask):\n",
    "        if self.need_birnn:\n",
    "            sequence_output, _ = self.birnn(sequence_output)\n",
    "        if self.need_cnn:\n",
    "            sequence_output=self.DGCNN([sequence_output.permute(0,2,1),attention_mask])[0].permute(0,2,1)\n",
    "        sequence_output = self.dropout(sequence_output)\n",
    "        emissions = self.hidden2tag(sequence_output)\n",
    "        return emissions\n",
    "def train_pointer(args, train_dataset,val_dataset, model, tokenizer):\n",
    "#     \"\"\" Train the model \"\"\"\n",
    "#     if args.local_rank in [-1, 0]:\n",
    "#         tb_writer = SummaryWriter()\n",
    "\n",
    "#     args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)\n",
    "    train_sampler = RandomSampler(train_dataset) # if args.local_rank == -1 else DistributedSampler(train_dataset)\n",
    "    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)\n",
    "\n",
    "    if args.max_steps > 0:\n",
    "        t_total = args.max_steps\n",
    "        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1\n",
    "    else:\n",
    "        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs\n",
    "\n",
    "    # Prepare optimizer and schedule (linear warmup and decay)\n",
    "    no_decay = [\"bias\", \"LayerNorm.weight\"]\n",
    "    optimizer_grouped_parameters = [\n",
    "        {\n",
    "            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
    "            \"weight_decay\": args.weight_decay,\n",
    "        },\n",
    "        {\"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], \"weight_decay\": 0.0},\n",
    "    ]\n",
    "\n",
    "    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)\n",
    "    scheduler = get_linear_schedule_with_warmup(\n",
    "        optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total\n",
    "    )\n",
    "\n",
    "    # Check if saved optimizer or scheduler states exist\n",
    "    if os.path.isfile(os.path.join(args.model_name_or_path, \"optimizer.pt\")) and os.path.isfile(\n",
    "        os.path.join(args.model_name_or_path, \"scheduler.pt\")\n",
    "    ):\n",
    "        logger.info(\"  loading optimizer and scheduler...\")\n",
    "        # Load in optimizer and scheduler states\n",
    "        optimizer.load_state_dict(torch.load(os.path.join(args.model_name_or_path, \"optimizer.pt\")))\n",
    "#         scheduler.load_state_dict(torch.load(os.path.join(args.model_name_or_path, \"scheduler.pt\")))\n",
    "    else:\n",
    "        logger.info(\"  No optimizer and scheduler,we build a new one\")        \n",
    "\n",
    "    if args.fp16:\n",
    "        try:\n",
    "            from apex import amp\n",
    "        except ImportError:\n",
    "            raise ImportError(\"Please install apex from https://www.github.com/nvidia/apex to use fp16 training.\")\n",
    "        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)\n",
    "\n",
    "    # multi-gpu training (should be after apex fp16 initialization)\n",
    "    if args.n_gpu > 1:\n",
    "        model = torch.nn.DataParallel(model,device_ids=args.card_list)\n",
    "\n",
    "\n",
    "    # Train!\n",
    "    logger.info(\"***** Running training *****\")\n",
    "    logger.info(\"  Num examples = %d\", len(train_dataset))\n",
    "    logger.info(\"  Num Epochs = %d\", args.num_train_epochs)\n",
    "#     logger.info(\"  Instantaneous batch size per GPU = %d\", args.per_gpu_train_batch_size)\n",
    "    logger.info(\n",
    "        \"  Total train batch size (w. parallel, distributed & accumulation) = %d\",\n",
    "        args.train_batch_size\n",
    "        * args.gradient_accumulation_steps\n",
    "#         * (torch.distributed.get_world_size() if args.local_rank != -1 else 1),\n",
    "    )\n",
    "    logger.info(\"  Gradient Accumulation steps = %d\", args.gradient_accumulation_steps)\n",
    "    logger.info(\"  Total optimization steps = %d\", t_total)\n",
    "\n",
    "    global_step = 0\n",
    "    epochs_trained = 0\n",
    "    steps_trained_in_current_epoch = 0\n",
    "    # Check if continuing training from a checkpoint\n",
    "    if os.path.exists(args.model_name_or_path):\n",
    "        # set global_step to global_step of last saved checkpoint from model path\n",
    "        try:\n",
    "            global_step = int(args.model_name_or_path.split(\"-\")[-1].split(\"/\")[0])\n",
    "        except ValueError:\n",
    "            global_step = 0\n",
    "        epochs_trained = global_step // (len(train_dataloader) // args.gradient_accumulation_steps)\n",
    "        steps_trained_in_current_epoch = global_step % (len(train_dataloader) // args.gradient_accumulation_steps)\n",
    "\n",
    "        logger.info(\"  Continuing training from checkpoint, will skip to saved global_step\")\n",
    "        logger.info(\"  Continuing training from epoch %d\", epochs_trained)\n",
    "        logger.info(\"  Continuing training from global step %d\", global_step)\n",
    "        logger.info(\"  Will skip the first %d steps in the first epoch\", steps_trained_in_current_epoch)\n",
    "\n",
    "    tr_loss, logging_loss = 0.0, 0.0\n",
    "    model.zero_grad()\n",
    "    train_iterator = tqdm(range(\n",
    "        epochs_trained, int(args.num_train_epochs)), desc=\"Epoch\")\n",
    "    set_seed(args)  # Added here for reproductibility\n",
    "    for _ in train_iterator:\n",
    "        epoch_iterator = tqdm(train_dataloader, desc=\"Iteration\")\n",
    "        for step, batch in enumerate(epoch_iterator):\n",
    "\n",
    "            # Skip past any already trained steps if resuming training\n",
    "            if steps_trained_in_current_epoch > 0:\n",
    "                if  (step + 1) % args.gradient_accumulation_steps == 0: \n",
    "                        steps_trained_in_current_epoch -= 1\n",
    "                continue\n",
    "\n",
    "            model.train()\n",
    "            batch = tuple(t.to(args.device) for t in batch[:-1])\n",
    "            inputs = {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3],\"label_tokens\":batch[4],\"plan_label\":batch[7].float()}\n",
    "            inputs[\"token_type_ids\"]=batch[2]\n",
    "#             if args.model_type != \"distilbert\":\n",
    "#                 inputs[\"token_type_ids\"] = (\n",
    "#                     batch[2] if args.model_type in [\"bert\", \"xlnet\", \"albert\"] else None\n",
    "#                 )  # XLM, DistilBERT, RoBERTa, and XLM-RoBERTa don't use segment_ids\n",
    "            outputs = model(**inputs)\n",
    "            loss = outputs[0]  # model outputs are always tuple in transformers (see doc)\n",
    "\n",
    "            if args.n_gpu > 1:\n",
    "                loss = loss.mean()  # mean() to average on multi-gpu parallel training\n",
    "            if args.gradient_accumulation_steps > 1:\n",
    "                loss = loss / args.gradient_accumulation_steps\n",
    "\n",
    "            if args.fp16:\n",
    "                with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "                    scaled_loss.backward()\n",
    "            else:\n",
    "                loss.backward()\n",
    "            logger.info(\"  step:%d loss %.3f\", step,loss.item())\n",
    "\n",
    "            tr_loss += loss.item()\n",
    "            if (step + 1) % args.gradient_accumulation_steps == 0:\n",
    "                if args.fp16:\n",
    "                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)\n",
    "                else:\n",
    "                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)\n",
    "\n",
    "                optimizer.step()\n",
    "                scheduler.step()  # Update learning rate schedule\n",
    "                model.zero_grad()\n",
    "                global_step += 1\n",
    "\n",
    "                if  args.save_steps > 0 and global_step % args.save_steps == 0:\n",
    "                    # Save model checkpoint\n",
    "                    results = evaluate_pointer(args, val_dataset,model, tokenizer)\n",
    "                    output_dir = os.path.join(args.output_dir, \"checkpoint-{}\".format(global_step))\n",
    "                    if not os.path.exists(output_dir):\n",
    "                        os.makedirs(output_dir)\n",
    "                    model_to_save = (\n",
    "                        model.module if hasattr(model, \"module\") else model\n",
    "                    )  # Take care of distributed/parallel training\n",
    "                    torch.save(model_to_save.state_dict(),os.path.join(output_dir,\"model.pt\"))\n",
    "                    tokenizer.save_pretrained(output_dir)\n",
    "\n",
    "                    torch.save(args, os.path.join(output_dir, \"training_args.bin\"))\n",
    "                    logger.info(\"Saving model checkpoint to %s\", output_dir)\n",
    "                    if args.fp16:\n",
    "                        torch.save(amp.state_dict(),os.path.join(output_dir, \"amp.pt\"))\n",
    "                    torch.save(optimizer.state_dict(), os.path.join(output_dir, \"optimizer.pt\"))\n",
    "                    torch.save(scheduler.state_dict(), os.path.join(output_dir, \"scheduler.pt\"))\n",
    "                    logger.info(\"Saving optimizer and scheduler states to %s\", output_dir)\n",
    "\n",
    "            if args.max_steps > 0 and global_step > args.max_steps:\n",
    "                epoch_iterator.close()\n",
    "                break\n",
    "        if args.max_steps > 0 and global_step > args.max_steps:\n",
    "            train_iterator.close()\n",
    "            break\n",
    "\n",
    "\n",
    "    return global_step, tr_loss / global_step\n",
    "def evaluate_pointer(args, eval_dataset,model, tokenizer, thre=0.5,prefix=\"\"):\n",
    "    eval_output_dir = args.output_dir \n",
    "\n",
    "    results = {}\n",
    "\n",
    "    if not os.path.exists(eval_output_dir) :\n",
    "        os.makedirs(eval_output_dir)\n",
    "\n",
    "    eval_sampler = SequentialSampler(eval_dataset)\n",
    "    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size,num_workers=0)\n",
    "\n",
    "    # multi-gpu eval\n",
    "#         if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):\n",
    "#             model = torch.nn.DataParallel(model)\n",
    "\n",
    "    # Eval!\n",
    "    logger.info(\"***** Running evaluation {} *****\".format(prefix))\n",
    "    logger.info(\"  Num examples = %d\", len(eval_dataset))\n",
    "    logger.info(\"  Batch size = %d\", args.eval_batch_size)\n",
    "    eval_loss = 0.0\n",
    "    nb_eval_steps = 0\n",
    "    rel_preds=None\n",
    "    rel_labels=None\n",
    "    token_preds=None\n",
    "    token_labels=None\n",
    "    recall_num=np.zeros((4,))\n",
    "    precision_num=np.zeros((4,))\n",
    "    full_num=np.zeros((4,))\n",
    "    full_num_pre=np.zeros((4,))\n",
    "    for batch in tqdm(eval_dataloader, desc=\"Evaluating\"):\n",
    "        model.eval()\n",
    "        batch = tuple(t.to(args.device) for t in batch[:-1])\n",
    "\n",
    "        with torch.no_grad():\n",
    "            inputs = {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"labels\": batch[3],\"label_tokens\":batch[4],\"plan_label\":batch[7].float()}\n",
    "            inputs[\"token_type_ids\"]=batch[2]\n",
    "#             temp_inputs={\"input_ids\": batch[0], \"attention_mask\": batch[1],\"token_type_ids\":batch[2]}\n",
    "#                 if args.model_type != \"distilbert\":\n",
    "#                     inputs[\"token_type_ids\"] = (\n",
    "#                         batch[2] if args.model_type in [\"bert\", \"xlnet\", \"albert\"] else None\n",
    "#                     )  # XLM, DistilBERT, RoBERTa, and XLM-RoBERTa don't use segment_ids\n",
    "            outputs = model(**inputs)\n",
    "            tmp_eval_loss, rel_logits, token_logits = outputs[:3]\n",
    "            \n",
    "            eval_loss += tmp_eval_loss.mean().item()\n",
    "        nb_eval_steps += 1\n",
    "        mask=batch[1].unsqueeze(-1)\n",
    "        rel_pred = F.softmax(rel_logits,dim=-1)\n",
    "        rel_pred =torch.argmax(rel_pred,dim=-1).detach().cpu().numpy()\n",
    "        rel_label=torch.argmax(inputs[\"labels\"],dim=-1).detach().cpu().numpy()\n",
    "        if rel_preds is None:\n",
    "            rel_preds=rel_pred\n",
    "        else:\n",
    "            rel_preds=np.append(rel_preds,rel_pred,axis=0)\n",
    "        if rel_labels is None:\n",
    "            rel_labels=rel_label\n",
    "        else:\n",
    "            rel_labels=np.append(rel_labels,rel_label,axis=0)     \n",
    "\n",
    "            \n",
    "\n",
    "        recall_num+=((((torch.sigmoid(token_logits.detach())>thre).long()==1) & (batch[4].detach()[:,:,1:5]!=0)).long()*mask).sum(dim=[0,1]).detach().cpu().numpy()\n",
    "        precision_num+=((((torch.sigmoid(token_logits.detach())>thre).long()==1) & (batch[4].detach()[:,:,1:5]!=0)).long()*mask).sum(dim=[0,1]).detach().cpu().numpy()\n",
    "        full_num+=((batch[4].detach()[:,:,1:5]!=0).long()*mask).sum(dim=[0,1]).detach().cpu().numpy()\n",
    "        full_num_pre+=((torch.sigmoid(token_logits.detach())>thre).long()*mask).sum(dim=[0,1]).detach().cpu().numpy()\n",
    "        token_pred=((torch.sigmoid(token_logits.detach())>thre).long()*mask).cpu().numpy().reshape(-1,4)\n",
    "        token_label=((batch[4].detach()[:,:,1:5]!=0).long()*mask).cpu().numpy().reshape(-1,4)\n",
    "        if token_preds is None:\n",
    "            token_preds=token_pred\n",
    "        else:\n",
    "            token_preds=np.append(token_preds,token_pred,axis=0)\n",
    "        if token_labels is None:\n",
    "            token_labels=token_label\n",
    "        else:\n",
    "            token_labels=np.append(token_labels,token_label,axis=0) \n",
    "    result = {\"recall_sample\":recall_num/(full_num+0.001),\"precision_sample\":precision_num/(full_num_pre+0.001)}\n",
    "    results.update(result)\n",
    "\n",
    "\n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    print(\"准召报告：\")\n",
    "    df_precison_recall_f1=pd.DataFrame(metrics.precision_recall_fscore_support(rel_labels,rel_preds),index=['precison','recall','f1','number'])\n",
    "    display(HTML(df_precison_recall_f1.to_html()))\n",
    "    result = {\"cls\":    df_precison_recall_f1}\n",
    "    results.update(result)\n",
    "\n",
    "    output_eval_file = os.path.join(eval_output_dir, prefix, \"eval_results.txt\")\n",
    "    with open(output_eval_file, \"a\") as writer:\n",
    "        logger.info(\"***** Eval results {} *****\".format(prefix))\n",
    "        for key in sorted(results.keys()):\n",
    "            logger.info(\"  %s = %s\", key, str(results[key]))\n",
    "            writer.write(\"%s = %s\\n\" % (key, str(results[key])))\n",
    "\n",
    "    return rel_labels,rel_preds,token_preds,token_labels\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/11/2020 16:21:15 - INFO - transformers.configuration_utils -   loading configuration file ./roberta-zh-wwm-pytorch/bert_config.json\n",
      "05/11/2020 16:21:15 - INFO - transformers.configuration_utils -   Model config BertConfig {\n",
      "  \"_num_labels\": 2,\n",
      "  \"architectures\": null,\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": null,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"do_sample\": false,\n",
      "  \"early_stopping\": false,\n",
      "  \"eos_token_ids\": null,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": true,\n",
      "  \"output_hidden_states\": true,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": null,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"pruned_heads\": {},\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"temperature\": 1.0,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 21128\n",
      "}\n",
      "\n",
      "05/11/2020 16:21:15 - INFO - transformers.modeling_utils -   loading weights file ./roberta-zh-wwm-pytorch/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "# model=BertSeqLabeling(bert_dir,config_file,num_rel_labels=num_rel_labels,num_token_labels=num_token_labels,\n",
    "#                                      hidden_dropout_prob=hidden_dropout_prob)\n",
    "# model=BERT_BiLSTM_CRF(bert_dir,config_file,num_rel_labels=num_rel_labels,num_token_labels=num_token_labels,\n",
    "#                                      hidden_dropout_prob=hidden_dropout_prob,need_birnn=False,need_cnn=True)\n",
    "model=BERT_BiLSTM_POINTER(bert_dir,config_file,num_rel_labels=num_rel_labels,num_token_labels=num_token_labels,\n",
    "                         hidden_dropout_prob=hidden_dropout_prob,need_birnn=need_rnn,need_cnn=need_cnn\\\n",
    "                          ,use_plan=use_plan,p=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "args=ARG(train_batch_size=batch_size*2,eval_batch_size=batch_size*2,weight_decay=weight_decay,learning_rate=learning_rate,\n",
    "         adam_epsilon=adam_epsilon,num_train_epochs=epochs,warmup_steps=0,gradient_accumulation_steps=1,save_steps=8389,\n",
    "         max_grad_norm=1.0,model_name_or_path=output_dir,output_dir=output_dir,seed=42,device=device,n_gpu=2,\n",
    "        max_steps=0,output_mode=\"classification\",fp16=False,fp16_opt_level='O1',card_list=card_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/11/2020 16:21:21 - INFO - __main__ -     No optimizer and scheduler,we build a new one\n",
      "05/11/2020 16:21:21 - INFO - __main__ -   ***** Running training *****\n",
      "05/11/2020 16:21:21 - INFO - __main__ -     Num examples = 10000\n",
      "05/11/2020 16:21:21 - INFO - __main__ -     Num Epochs = 3\n",
      "05/11/2020 16:21:21 - INFO - __main__ -     Total train batch size (w. parallel, distributed & accumulation) = 32\n",
      "05/11/2020 16:21:21 - INFO - __main__ -     Gradient Accumulation steps = 1\n",
      "05/11/2020 16:21:21 - INFO - __main__ -     Total optimization steps = 939\n",
      "05/11/2020 16:21:21 - INFO - __main__ -     Continuing training from checkpoint, will skip to saved global_step\n",
      "05/11/2020 16:21:21 - INFO - __main__ -     Continuing training from epoch 0\n",
      "05/11/2020 16:21:21 - INFO - __main__ -     Continuing training from global step 0\n",
      "05/11/2020 16:21:21 - INFO - __main__ -     Will skip the first 0 steps in the first epoch\n",
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/ipykernel_launcher.py:606: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47fb198c94904006b5d1fed7dbb334f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=3, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/ipykernel_launcher.py:609: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b693dd8407b423293a3c011ccf57ef5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration', max=313, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:61: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "05/11/2020 16:21:26 - INFO - __main__ -     step:0 loss 74.139\n",
      "05/11/2020 16:21:27 - INFO - __main__ -     step:1 loss 71.103\n",
      "05/11/2020 16:21:28 - INFO - __main__ -     step:2 loss 69.674\n",
      "05/11/2020 16:21:29 - INFO - __main__ -     step:3 loss 62.928\n",
      "05/11/2020 16:21:30 - INFO - __main__ -     step:4 loss 63.715\n",
      "05/11/2020 16:21:31 - INFO - __main__ -     step:5 loss 61.972\n",
      "05/11/2020 16:21:32 - INFO - __main__ -     step:6 loss 55.312\n",
      "05/11/2020 16:21:33 - INFO - __main__ -     step:7 loss 54.271\n",
      "05/11/2020 16:21:34 - INFO - __main__ -     step:8 loss 54.373\n",
      "05/11/2020 16:21:35 - INFO - __main__ -     step:9 loss 53.113\n",
      "05/11/2020 16:21:36 - INFO - __main__ -     step:10 loss 50.606\n",
      "05/11/2020 16:21:37 - INFO - __main__ -     step:11 loss 48.948\n",
      "05/11/2020 16:21:38 - INFO - __main__ -     step:12 loss 49.995\n",
      "05/11/2020 16:21:39 - INFO - __main__ -     step:13 loss 50.260\n",
      "05/11/2020 16:21:40 - INFO - __main__ -     step:14 loss 49.985\n",
      "05/11/2020 16:21:41 - INFO - __main__ -     step:15 loss 47.291\n",
      "05/11/2020 16:21:42 - INFO - __main__ -     step:16 loss 45.606\n",
      "05/11/2020 16:21:43 - INFO - __main__ -     step:17 loss 48.277\n",
      "05/11/2020 16:21:44 - INFO - __main__ -     step:18 loss 44.556\n",
      "05/11/2020 16:21:45 - INFO - __main__ -     step:19 loss 45.255\n",
      "05/11/2020 16:21:46 - INFO - __main__ -     step:20 loss 43.846\n",
      "05/11/2020 16:21:47 - INFO - __main__ -     step:21 loss 44.654\n",
      "05/11/2020 16:21:48 - INFO - __main__ -     step:22 loss 39.723\n",
      "05/11/2020 16:21:49 - INFO - __main__ -     step:23 loss 40.181\n",
      "05/11/2020 16:21:50 - INFO - __main__ -     step:24 loss 38.515\n",
      "05/11/2020 16:21:50 - INFO - __main__ -     step:25 loss 37.707\n",
      "05/11/2020 16:21:51 - INFO - __main__ -     step:26 loss 38.777\n",
      "05/11/2020 16:21:52 - INFO - __main__ -     step:27 loss 37.242\n",
      "05/11/2020 16:21:53 - INFO - __main__ -     step:28 loss 36.171\n",
      "05/11/2020 16:21:54 - INFO - __main__ -     step:29 loss 36.636\n",
      "05/11/2020 16:21:55 - INFO - __main__ -     step:30 loss 34.247\n",
      "05/11/2020 16:21:56 - INFO - __main__ -     step:31 loss 35.439\n",
      "05/11/2020 16:21:57 - INFO - __main__ -     step:32 loss 35.143\n",
      "05/11/2020 16:21:58 - INFO - __main__ -     step:33 loss 34.332\n",
      "05/11/2020 16:21:59 - INFO - __main__ -     step:34 loss 32.193\n",
      "05/11/2020 16:22:00 - INFO - __main__ -     step:35 loss 31.809\n",
      "05/11/2020 16:22:01 - INFO - __main__ -     step:36 loss 30.734\n",
      "05/11/2020 16:22:02 - INFO - __main__ -     step:37 loss 30.718\n",
      "05/11/2020 16:22:03 - INFO - __main__ -     step:38 loss 29.932\n",
      "05/11/2020 16:22:04 - INFO - __main__ -     step:39 loss 29.817\n",
      "05/11/2020 16:22:05 - INFO - __main__ -     step:40 loss 33.193\n",
      "05/11/2020 16:22:06 - INFO - __main__ -     step:41 loss 28.420\n",
      "05/11/2020 16:22:07 - INFO - __main__ -     step:42 loss 27.249\n",
      "05/11/2020 16:22:08 - INFO - __main__ -     step:43 loss 26.383\n",
      "05/11/2020 16:22:09 - INFO - __main__ -     step:44 loss 27.867\n",
      "05/11/2020 16:22:10 - INFO - __main__ -     step:45 loss 27.022\n",
      "05/11/2020 16:22:11 - INFO - __main__ -     step:46 loss 27.031\n",
      "05/11/2020 16:22:12 - INFO - __main__ -     step:47 loss 26.218\n",
      "05/11/2020 16:22:13 - INFO - __main__ -     step:48 loss 26.788\n",
      "05/11/2020 16:22:14 - INFO - __main__ -     step:49 loss 26.307\n",
      "05/11/2020 16:22:15 - INFO - __main__ -     step:50 loss 24.217\n",
      "05/11/2020 16:22:16 - INFO - __main__ -     step:51 loss 25.021\n",
      "05/11/2020 16:22:17 - INFO - __main__ -     step:52 loss 25.663\n",
      "05/11/2020 16:22:18 - INFO - __main__ -     step:53 loss 27.084\n",
      "05/11/2020 16:22:19 - INFO - __main__ -     step:54 loss 24.699\n",
      "05/11/2020 16:22:20 - INFO - __main__ -     step:55 loss 22.519\n",
      "05/11/2020 16:22:21 - INFO - __main__ -     step:56 loss 23.270\n",
      "05/11/2020 16:22:22 - INFO - __main__ -     step:57 loss 23.214\n",
      "05/11/2020 16:22:23 - INFO - __main__ -     step:58 loss 23.123\n",
      "05/11/2020 16:22:24 - INFO - __main__ -     step:59 loss 22.248\n",
      "05/11/2020 16:22:25 - INFO - __main__ -     step:60 loss 22.575\n",
      "05/11/2020 16:22:26 - INFO - __main__ -     step:61 loss 22.520\n",
      "05/11/2020 16:22:27 - INFO - __main__ -     step:62 loss 22.522\n",
      "05/11/2020 16:22:28 - INFO - __main__ -     step:63 loss 21.319\n",
      "05/11/2020 16:22:29 - INFO - __main__ -     step:64 loss 21.180\n",
      "05/11/2020 16:22:30 - INFO - __main__ -     step:65 loss 20.685\n",
      "05/11/2020 16:22:31 - INFO - __main__ -     step:66 loss 20.803\n",
      "05/11/2020 16:22:32 - INFO - __main__ -     step:67 loss 19.838\n",
      "05/11/2020 16:22:33 - INFO - __main__ -     step:68 loss 22.231\n",
      "05/11/2020 16:22:34 - INFO - __main__ -     step:69 loss 22.384\n",
      "05/11/2020 16:22:35 - INFO - __main__ -     step:70 loss 20.906\n",
      "05/11/2020 16:22:36 - INFO - __main__ -     step:71 loss 20.704\n",
      "05/11/2020 16:22:37 - INFO - __main__ -     step:72 loss 20.939\n",
      "05/11/2020 16:22:38 - INFO - __main__ -     step:73 loss 19.361\n",
      "05/11/2020 16:22:39 - INFO - __main__ -     step:74 loss 18.988\n",
      "05/11/2020 16:22:40 - INFO - __main__ -     step:75 loss 19.923\n",
      "05/11/2020 16:22:41 - INFO - __main__ -     step:76 loss 23.188\n",
      "05/11/2020 16:22:42 - INFO - __main__ -     step:77 loss 19.363\n",
      "05/11/2020 16:22:43 - INFO - __main__ -     step:78 loss 19.143\n",
      "05/11/2020 16:22:44 - INFO - __main__ -     step:79 loss 19.422\n",
      "05/11/2020 16:22:45 - INFO - __main__ -     step:80 loss 18.832\n",
      "05/11/2020 16:22:46 - INFO - __main__ -     step:81 loss 18.517\n",
      "05/11/2020 16:22:47 - INFO - __main__ -     step:82 loss 19.755\n",
      "05/11/2020 16:22:48 - INFO - __main__ -     step:83 loss 17.287\n",
      "05/11/2020 16:22:49 - INFO - __main__ -     step:84 loss 17.217\n",
      "05/11/2020 16:22:50 - INFO - __main__ -     step:85 loss 19.394\n",
      "05/11/2020 16:22:51 - INFO - __main__ -     step:86 loss 17.975\n",
      "05/11/2020 16:22:52 - INFO - __main__ -     step:87 loss 19.912\n",
      "05/11/2020 16:22:53 - INFO - __main__ -     step:88 loss 17.407\n",
      "05/11/2020 16:22:54 - INFO - __main__ -     step:89 loss 18.279\n",
      "05/11/2020 16:22:55 - INFO - __main__ -     step:90 loss 18.996\n",
      "05/11/2020 16:22:56 - INFO - __main__ -     step:91 loss 20.641\n",
      "05/11/2020 16:22:57 - INFO - __main__ -     step:92 loss 17.704\n",
      "05/11/2020 16:22:58 - INFO - __main__ -     step:93 loss 19.736\n",
      "05/11/2020 16:22:59 - INFO - __main__ -     step:94 loss 17.018\n",
      "05/11/2020 16:23:00 - INFO - __main__ -     step:95 loss 19.176\n",
      "05/11/2020 16:23:01 - INFO - __main__ -     step:96 loss 17.365\n",
      "05/11/2020 16:23:02 - INFO - __main__ -     step:97 loss 17.198\n",
      "05/11/2020 16:23:03 - INFO - __main__ -     step:98 loss 20.103\n",
      "05/11/2020 16:23:04 - INFO - __main__ -     step:99 loss 17.519\n",
      "05/11/2020 16:23:05 - INFO - __main__ -     step:100 loss 15.335\n",
      "05/11/2020 16:23:06 - INFO - __main__ -     step:101 loss 14.413\n",
      "05/11/2020 16:23:07 - INFO - __main__ -     step:102 loss 15.572\n",
      "05/11/2020 16:23:08 - INFO - __main__ -     step:103 loss 17.625\n",
      "05/11/2020 16:23:10 - INFO - __main__ -     step:104 loss 21.727\n",
      "05/11/2020 16:23:11 - INFO - __main__ -     step:105 loss 16.367\n",
      "05/11/2020 16:23:12 - INFO - __main__ -     step:106 loss 18.197\n",
      "05/11/2020 16:23:13 - INFO - __main__ -     step:107 loss 16.316\n",
      "05/11/2020 16:23:14 - INFO - __main__ -     step:108 loss 15.353\n",
      "05/11/2020 16:23:15 - INFO - __main__ -     step:109 loss 16.585\n",
      "05/11/2020 16:23:16 - INFO - __main__ -     step:110 loss 16.763\n",
      "05/11/2020 16:23:17 - INFO - __main__ -     step:111 loss 14.254\n",
      "05/11/2020 16:23:18 - INFO - __main__ -     step:112 loss 16.718\n",
      "05/11/2020 16:23:19 - INFO - __main__ -     step:113 loss 14.682\n",
      "05/11/2020 16:23:20 - INFO - __main__ -     step:114 loss 15.682\n",
      "05/11/2020 16:23:21 - INFO - __main__ -     step:115 loss 15.702\n",
      "05/11/2020 16:23:22 - INFO - __main__ -     step:116 loss 15.758\n",
      "05/11/2020 16:23:23 - INFO - __main__ -     step:117 loss 13.750\n",
      "05/11/2020 16:23:24 - INFO - __main__ -     step:118 loss 14.841\n",
      "05/11/2020 16:23:25 - INFO - __main__ -     step:119 loss 13.425\n",
      "05/11/2020 16:23:26 - INFO - __main__ -     step:120 loss 16.012\n",
      "05/11/2020 16:23:27 - INFO - __main__ -     step:121 loss 14.332\n",
      "05/11/2020 16:23:28 - INFO - __main__ -     step:122 loss 13.185\n",
      "05/11/2020 16:23:29 - INFO - __main__ -     step:123 loss 15.201\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/11/2020 16:23:30 - INFO - __main__ -     step:124 loss 14.133\n",
      "05/11/2020 16:23:31 - INFO - __main__ -     step:125 loss 14.598\n",
      "05/11/2020 16:23:32 - INFO - __main__ -     step:126 loss 15.484\n",
      "05/11/2020 16:23:33 - INFO - __main__ -     step:127 loss 13.207\n",
      "05/11/2020 16:23:34 - INFO - __main__ -     step:128 loss 13.181\n",
      "05/11/2020 16:23:35 - INFO - __main__ -     step:129 loss 13.894\n",
      "05/11/2020 16:23:36 - INFO - __main__ -     step:130 loss 15.606\n",
      "05/11/2020 16:23:37 - INFO - __main__ -     step:131 loss 15.933\n",
      "05/11/2020 16:23:38 - INFO - __main__ -     step:132 loss 12.671\n",
      "05/11/2020 16:23:39 - INFO - __main__ -     step:133 loss 13.170\n",
      "05/11/2020 16:23:40 - INFO - __main__ -     step:134 loss 15.012\n",
      "05/11/2020 16:23:41 - INFO - __main__ -     step:135 loss 14.803\n",
      "05/11/2020 16:23:42 - INFO - __main__ -     step:136 loss 14.295\n",
      "05/11/2020 16:23:43 - INFO - __main__ -     step:137 loss 14.036\n",
      "05/11/2020 16:23:44 - INFO - __main__ -     step:138 loss 13.856\n",
      "05/11/2020 16:23:45 - INFO - __main__ -     step:139 loss 12.272\n",
      "05/11/2020 16:23:46 - INFO - __main__ -     step:140 loss 13.733\n",
      "05/11/2020 16:23:47 - INFO - __main__ -     step:141 loss 13.777\n",
      "05/11/2020 16:23:48 - INFO - __main__ -     step:142 loss 12.866\n",
      "05/11/2020 16:23:50 - INFO - __main__ -     step:143 loss 13.519\n",
      "05/11/2020 16:23:51 - INFO - __main__ -     step:144 loss 14.972\n",
      "05/11/2020 16:23:52 - INFO - __main__ -     step:145 loss 12.691\n",
      "05/11/2020 16:23:53 - INFO - __main__ -     step:146 loss 14.109\n",
      "05/11/2020 16:23:54 - INFO - __main__ -     step:147 loss 12.709\n",
      "05/11/2020 16:23:55 - INFO - __main__ -     step:148 loss 12.934\n",
      "05/11/2020 16:23:56 - INFO - __main__ -     step:149 loss 13.487\n",
      "05/11/2020 16:23:57 - INFO - __main__ -     step:150 loss 12.514\n",
      "05/11/2020 16:23:58 - INFO - __main__ -     step:151 loss 12.894\n",
      "05/11/2020 16:23:59 - INFO - __main__ -     step:152 loss 12.869\n",
      "05/11/2020 16:24:00 - INFO - __main__ -     step:153 loss 14.432\n",
      "05/11/2020 16:24:01 - INFO - __main__ -     step:154 loss 13.008\n",
      "05/11/2020 16:24:02 - INFO - __main__ -     step:155 loss 13.240\n",
      "05/11/2020 16:24:03 - INFO - __main__ -     step:156 loss 11.995\n",
      "05/11/2020 16:24:04 - INFO - __main__ -     step:157 loss 13.259\n",
      "05/11/2020 16:24:05 - INFO - __main__ -     step:158 loss 13.840\n",
      "05/11/2020 16:24:06 - INFO - __main__ -     step:159 loss 11.829\n",
      "05/11/2020 16:24:07 - INFO - __main__ -     step:160 loss 11.445\n",
      "05/11/2020 16:24:08 - INFO - __main__ -     step:161 loss 14.326\n",
      "05/11/2020 16:24:09 - INFO - __main__ -     step:162 loss 11.519\n",
      "05/11/2020 16:24:10 - INFO - __main__ -     step:163 loss 12.021\n",
      "05/11/2020 16:24:11 - INFO - __main__ -     step:164 loss 11.885\n",
      "05/11/2020 16:24:12 - INFO - __main__ -     step:165 loss 11.265\n",
      "05/11/2020 16:24:13 - INFO - __main__ -     step:166 loss 12.126\n",
      "05/11/2020 16:24:14 - INFO - __main__ -     step:167 loss 12.352\n",
      "05/11/2020 16:24:16 - INFO - __main__ -     step:168 loss 11.474\n",
      "05/11/2020 16:24:17 - INFO - __main__ -     step:169 loss 13.486\n",
      "05/11/2020 16:24:18 - INFO - __main__ -     step:170 loss 11.165\n",
      "05/11/2020 16:24:19 - INFO - __main__ -     step:171 loss 11.706\n",
      "05/11/2020 16:24:20 - INFO - __main__ -     step:172 loss 11.674\n",
      "05/11/2020 16:24:21 - INFO - __main__ -     step:173 loss 9.589\n",
      "05/11/2020 16:24:22 - INFO - __main__ -     step:174 loss 13.694\n",
      "05/11/2020 16:24:23 - INFO - __main__ -     step:175 loss 11.144\n",
      "05/11/2020 16:24:24 - INFO - __main__ -     step:176 loss 11.442\n",
      "05/11/2020 16:24:25 - INFO - __main__ -     step:177 loss 12.156\n",
      "05/11/2020 16:24:26 - INFO - __main__ -     step:178 loss 12.046\n",
      "05/11/2020 16:24:27 - INFO - __main__ -     step:179 loss 14.006\n",
      "05/11/2020 16:24:28 - INFO - __main__ -     step:180 loss 13.338\n",
      "05/11/2020 16:24:29 - INFO - __main__ -     step:181 loss 12.262\n",
      "05/11/2020 16:24:30 - INFO - __main__ -     step:182 loss 11.254\n",
      "05/11/2020 16:24:31 - INFO - __main__ -     step:183 loss 10.540\n",
      "05/11/2020 16:24:32 - INFO - __main__ -     step:184 loss 12.255\n",
      "05/11/2020 16:24:33 - INFO - __main__ -     step:185 loss 11.676\n",
      "05/11/2020 16:24:34 - INFO - __main__ -     step:186 loss 10.015\n",
      "05/11/2020 16:24:35 - INFO - __main__ -     step:187 loss 10.221\n",
      "05/11/2020 16:24:36 - INFO - __main__ -     step:188 loss 13.362\n",
      "05/11/2020 16:24:37 - INFO - __main__ -     step:189 loss 12.281\n",
      "05/11/2020 16:24:38 - INFO - __main__ -     step:190 loss 13.547\n",
      "05/11/2020 16:24:39 - INFO - __main__ -     step:191 loss 11.297\n",
      "05/11/2020 16:24:40 - INFO - __main__ -     step:192 loss 10.979\n",
      "05/11/2020 16:24:41 - INFO - __main__ -     step:193 loss 12.547\n",
      "05/11/2020 16:24:42 - INFO - __main__ -     step:194 loss 11.862\n",
      "05/11/2020 16:24:43 - INFO - __main__ -     step:195 loss 11.859\n",
      "05/11/2020 16:24:44 - INFO - __main__ -     step:196 loss 11.666\n",
      "05/11/2020 16:24:45 - INFO - __main__ -     step:197 loss 10.348\n",
      "05/11/2020 16:24:46 - INFO - __main__ -     step:198 loss 11.987\n",
      "05/11/2020 16:24:47 - INFO - __main__ -     step:199 loss 11.183\n",
      "05/11/2020 16:24:48 - INFO - __main__ -     step:200 loss 11.129\n",
      "05/11/2020 16:24:49 - INFO - __main__ -     step:201 loss 11.890\n",
      "05/11/2020 16:24:50 - INFO - __main__ -     step:202 loss 11.564\n",
      "05/11/2020 16:24:52 - INFO - __main__ -     step:203 loss 10.746\n",
      "05/11/2020 16:24:53 - INFO - __main__ -     step:204 loss 12.332\n",
      "05/11/2020 16:24:54 - INFO - __main__ -     step:205 loss 10.535\n",
      "05/11/2020 16:24:55 - INFO - __main__ -     step:206 loss 10.749\n",
      "05/11/2020 16:24:56 - INFO - __main__ -     step:207 loss 11.184\n",
      "05/11/2020 16:24:57 - INFO - __main__ -     step:208 loss 11.592\n",
      "05/11/2020 16:24:58 - INFO - __main__ -     step:209 loss 10.619\n",
      "05/11/2020 16:24:59 - INFO - __main__ -     step:210 loss 10.023\n",
      "05/11/2020 16:25:00 - INFO - __main__ -     step:211 loss 11.611\n",
      "05/11/2020 16:25:01 - INFO - __main__ -     step:212 loss 12.956\n",
      "05/11/2020 16:25:02 - INFO - __main__ -     step:213 loss 10.368\n",
      "05/11/2020 16:25:03 - INFO - __main__ -     step:214 loss 12.834\n",
      "05/11/2020 16:25:04 - INFO - __main__ -     step:215 loss 10.566\n",
      "05/11/2020 16:25:05 - INFO - __main__ -     step:216 loss 10.126\n",
      "05/11/2020 16:25:06 - INFO - __main__ -     step:217 loss 11.155\n",
      "05/11/2020 16:25:07 - INFO - __main__ -     step:218 loss 10.341\n",
      "05/11/2020 16:25:08 - INFO - __main__ -     step:219 loss 10.159\n",
      "05/11/2020 16:25:09 - INFO - __main__ -     step:220 loss 10.925\n",
      "05/11/2020 16:25:10 - INFO - __main__ -     step:221 loss 11.714\n",
      "05/11/2020 16:25:11 - INFO - __main__ -     step:222 loss 9.076\n",
      "05/11/2020 16:25:12 - INFO - __main__ -     step:223 loss 10.539\n",
      "05/11/2020 16:25:13 - INFO - __main__ -     step:224 loss 12.186\n",
      "05/11/2020 16:25:14 - INFO - __main__ -     step:225 loss 9.342\n",
      "05/11/2020 16:25:15 - INFO - __main__ -     step:226 loss 9.060\n",
      "05/11/2020 16:25:16 - INFO - __main__ -     step:227 loss 9.859\n",
      "05/11/2020 16:25:17 - INFO - __main__ -     step:228 loss 10.709\n",
      "05/11/2020 16:25:18 - INFO - __main__ -     step:229 loss 10.242\n",
      "05/11/2020 16:25:19 - INFO - __main__ -     step:230 loss 11.247\n",
      "05/11/2020 16:25:20 - INFO - __main__ -     step:231 loss 10.976\n",
      "05/11/2020 16:25:21 - INFO - __main__ -     step:232 loss 10.156\n",
      "05/11/2020 16:25:22 - INFO - __main__ -     step:233 loss 11.190\n",
      "05/11/2020 16:25:23 - INFO - __main__ -     step:234 loss 9.666\n",
      "05/11/2020 16:25:24 - INFO - __main__ -     step:235 loss 9.609\n",
      "05/11/2020 16:25:25 - INFO - __main__ -     step:236 loss 8.444\n",
      "05/11/2020 16:25:26 - INFO - __main__ -     step:237 loss 11.544\n",
      "05/11/2020 16:25:27 - INFO - __main__ -     step:238 loss 9.230\n",
      "05/11/2020 16:25:28 - INFO - __main__ -     step:239 loss 10.345\n",
      "05/11/2020 16:25:29 - INFO - __main__ -     step:240 loss 8.328\n",
      "05/11/2020 16:25:30 - INFO - __main__ -     step:241 loss 9.314\n",
      "05/11/2020 16:25:31 - INFO - __main__ -     step:242 loss 10.795\n",
      "05/11/2020 16:25:32 - INFO - __main__ -     step:243 loss 12.593\n",
      "05/11/2020 16:25:34 - INFO - __main__ -     step:244 loss 9.481\n",
      "05/11/2020 16:25:35 - INFO - __main__ -     step:245 loss 10.026\n",
      "05/11/2020 16:25:36 - INFO - __main__ -     step:246 loss 12.349\n",
      "05/11/2020 16:25:37 - INFO - __main__ -     step:247 loss 9.451\n",
      "05/11/2020 16:25:38 - INFO - __main__ -     step:248 loss 11.801\n",
      "05/11/2020 16:25:39 - INFO - __main__ -     step:249 loss 9.786\n",
      "05/11/2020 16:25:40 - INFO - __main__ -     step:250 loss 10.515\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/11/2020 16:25:41 - INFO - __main__ -     step:251 loss 9.937\n",
      "05/11/2020 16:25:43 - INFO - __main__ -     step:252 loss 10.693\n",
      "05/11/2020 16:25:44 - INFO - __main__ -     step:253 loss 10.071\n",
      "05/11/2020 16:25:45 - INFO - __main__ -     step:254 loss 9.713\n",
      "05/11/2020 16:25:46 - INFO - __main__ -     step:255 loss 8.979\n",
      "05/11/2020 16:25:47 - INFO - __main__ -     step:256 loss 9.876\n",
      "05/11/2020 16:25:48 - INFO - __main__ -     step:257 loss 11.316\n",
      "05/11/2020 16:25:49 - INFO - __main__ -     step:258 loss 9.990\n",
      "05/11/2020 16:25:50 - INFO - __main__ -     step:259 loss 11.085\n",
      "05/11/2020 16:25:51 - INFO - __main__ -     step:260 loss 10.597\n",
      "05/11/2020 16:25:52 - INFO - __main__ -     step:261 loss 9.759\n",
      "05/11/2020 16:25:53 - INFO - __main__ -     step:262 loss 11.049\n",
      "05/11/2020 16:25:54 - INFO - __main__ -     step:263 loss 10.018\n",
      "05/11/2020 16:25:55 - INFO - __main__ -     step:264 loss 10.815\n",
      "05/11/2020 16:25:56 - INFO - __main__ -     step:265 loss 10.264\n",
      "05/11/2020 16:25:57 - INFO - __main__ -     step:266 loss 8.521\n",
      "05/11/2020 16:25:58 - INFO - __main__ -     step:267 loss 10.219\n",
      "05/11/2020 16:25:59 - INFO - __main__ -     step:268 loss 10.127\n",
      "05/11/2020 16:26:00 - INFO - __main__ -     step:269 loss 13.202\n",
      "05/11/2020 16:26:01 - INFO - __main__ -     step:270 loss 10.742\n",
      "05/11/2020 16:26:02 - INFO - __main__ -     step:271 loss 10.413\n",
      "05/11/2020 16:26:03 - INFO - __main__ -     step:272 loss 9.286\n",
      "05/11/2020 16:26:04 - INFO - __main__ -     step:273 loss 10.020\n",
      "05/11/2020 16:26:05 - INFO - __main__ -     step:274 loss 10.148\n",
      "05/11/2020 16:26:06 - INFO - __main__ -     step:275 loss 10.371\n",
      "05/11/2020 16:26:08 - INFO - __main__ -     step:276 loss 10.246\n",
      "05/11/2020 16:26:09 - INFO - __main__ -     step:277 loss 9.604\n",
      "05/11/2020 16:26:10 - INFO - __main__ -     step:278 loss 8.611\n",
      "05/11/2020 16:26:11 - INFO - __main__ -     step:279 loss 11.697\n",
      "05/11/2020 16:26:12 - INFO - __main__ -     step:280 loss 9.524\n",
      "05/11/2020 16:26:13 - INFO - __main__ -     step:281 loss 10.063\n",
      "05/11/2020 16:26:14 - INFO - __main__ -     step:282 loss 9.273\n",
      "05/11/2020 16:26:15 - INFO - __main__ -     step:283 loss 8.368\n",
      "05/11/2020 16:26:16 - INFO - __main__ -     step:284 loss 9.098\n",
      "05/11/2020 16:26:17 - INFO - __main__ -     step:285 loss 9.432\n",
      "05/11/2020 16:26:18 - INFO - __main__ -     step:286 loss 8.685\n",
      "05/11/2020 16:26:19 - INFO - __main__ -     step:287 loss 8.329\n",
      "05/11/2020 16:26:20 - INFO - __main__ -     step:288 loss 9.927\n",
      "05/11/2020 16:26:21 - INFO - __main__ -     step:289 loss 8.662\n",
      "05/11/2020 16:26:22 - INFO - __main__ -     step:290 loss 10.869\n",
      "05/11/2020 16:26:23 - INFO - __main__ -     step:291 loss 10.004\n",
      "05/11/2020 16:26:24 - INFO - __main__ -     step:292 loss 8.944\n",
      "05/11/2020 16:26:25 - INFO - __main__ -     step:293 loss 10.724\n",
      "05/11/2020 16:26:26 - INFO - __main__ -     step:294 loss 8.026\n",
      "05/11/2020 16:26:27 - INFO - __main__ -     step:295 loss 12.038\n",
      "05/11/2020 16:26:28 - INFO - __main__ -     step:296 loss 9.251\n",
      "05/11/2020 16:26:29 - INFO - __main__ -     step:297 loss 9.793\n",
      "05/11/2020 16:26:30 - INFO - __main__ -     step:298 loss 9.184\n",
      "05/11/2020 16:26:31 - INFO - __main__ -     step:299 loss 10.431\n",
      "05/11/2020 16:26:32 - INFO - __main__ -     step:300 loss 8.947\n",
      "05/11/2020 16:26:33 - INFO - __main__ -     step:301 loss 9.333\n",
      "05/11/2020 16:26:34 - INFO - __main__ -     step:302 loss 10.182\n",
      "05/11/2020 16:26:35 - INFO - __main__ -     step:303 loss 10.778\n",
      "05/11/2020 16:26:36 - INFO - __main__ -     step:304 loss 9.841\n",
      "05/11/2020 16:26:37 - INFO - __main__ -     step:305 loss 9.493\n",
      "05/11/2020 16:26:38 - INFO - __main__ -     step:306 loss 14.422\n",
      "05/11/2020 16:26:39 - INFO - __main__ -     step:307 loss 10.169\n",
      "05/11/2020 16:26:41 - INFO - __main__ -     step:308 loss 9.541\n",
      "05/11/2020 16:26:42 - INFO - __main__ -     step:309 loss 8.037\n",
      "05/11/2020 16:26:43 - INFO - __main__ -     step:310 loss 10.178\n",
      "05/11/2020 16:26:44 - INFO - __main__ -     step:311 loss 8.563\n",
      "05/11/2020 16:26:44 - INFO - __main__ -     step:312 loss 9.059\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f51672ee5e043aba2c804df322ca458",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration', max=313, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/11/2020 16:26:45 - INFO - __main__ -     step:0 loss 9.098\n",
      "05/11/2020 16:26:46 - INFO - __main__ -     step:1 loss 9.499\n",
      "05/11/2020 16:26:47 - INFO - __main__ -     step:2 loss 9.870\n",
      "05/11/2020 16:26:48 - INFO - __main__ -     step:3 loss 9.209\n",
      "05/11/2020 16:26:49 - INFO - __main__ -     step:4 loss 10.068\n",
      "05/11/2020 16:26:51 - INFO - __main__ -     step:5 loss 8.181\n",
      "05/11/2020 16:26:52 - INFO - __main__ -     step:6 loss 8.922\n",
      "05/11/2020 16:26:53 - INFO - __main__ -     step:7 loss 8.935\n",
      "05/11/2020 16:26:54 - INFO - __main__ -     step:8 loss 8.119\n",
      "05/11/2020 16:26:55 - INFO - __main__ -     step:9 loss 8.985\n",
      "05/11/2020 16:26:56 - INFO - __main__ -     step:10 loss 9.586\n",
      "05/11/2020 16:26:57 - INFO - __main__ -     step:11 loss 8.236\n",
      "05/11/2020 16:26:58 - INFO - __main__ -     step:12 loss 7.582\n",
      "05/11/2020 16:26:59 - INFO - __main__ -     step:13 loss 8.631\n",
      "05/11/2020 16:27:00 - INFO - __main__ -     step:14 loss 10.328\n",
      "05/11/2020 16:27:01 - INFO - __main__ -     step:15 loss 8.925\n",
      "05/11/2020 16:27:02 - INFO - __main__ -     step:16 loss 9.703\n",
      "05/11/2020 16:27:03 - INFO - __main__ -     step:17 loss 9.919\n",
      "05/11/2020 16:27:04 - INFO - __main__ -     step:18 loss 9.537\n",
      "05/11/2020 16:27:05 - INFO - __main__ -     step:19 loss 8.557\n",
      "05/11/2020 16:27:06 - INFO - __main__ -     step:20 loss 7.543\n",
      "05/11/2020 16:27:07 - INFO - __main__ -     step:21 loss 6.944\n",
      "05/11/2020 16:27:08 - INFO - __main__ -     step:22 loss 7.784\n",
      "05/11/2020 16:27:09 - INFO - __main__ -     step:23 loss 7.401\n",
      "05/11/2020 16:27:10 - INFO - __main__ -     step:24 loss 10.370\n",
      "05/11/2020 16:27:11 - INFO - __main__ -     step:25 loss 8.967\n",
      "05/11/2020 16:27:12 - INFO - __main__ -     step:26 loss 8.435\n",
      "05/11/2020 16:27:13 - INFO - __main__ -     step:27 loss 7.905\n",
      "05/11/2020 16:27:14 - INFO - __main__ -     step:28 loss 8.700\n",
      "05/11/2020 16:27:15 - INFO - __main__ -     step:29 loss 10.125\n",
      "05/11/2020 16:27:17 - INFO - __main__ -     step:30 loss 8.173\n",
      "05/11/2020 16:27:18 - INFO - __main__ -     step:31 loss 9.133\n",
      "05/11/2020 16:27:19 - INFO - __main__ -     step:32 loss 8.004\n",
      "05/11/2020 16:27:20 - INFO - __main__ -     step:33 loss 8.586\n",
      "05/11/2020 16:27:21 - INFO - __main__ -     step:34 loss 9.324\n",
      "05/11/2020 16:27:22 - INFO - __main__ -     step:35 loss 8.727\n",
      "05/11/2020 16:27:23 - INFO - __main__ -     step:36 loss 8.780\n",
      "05/11/2020 16:27:24 - INFO - __main__ -     step:37 loss 9.865\n",
      "05/11/2020 16:27:25 - INFO - __main__ -     step:38 loss 8.715\n",
      "05/11/2020 16:27:26 - INFO - __main__ -     step:39 loss 7.762\n",
      "05/11/2020 16:27:27 - INFO - __main__ -     step:40 loss 7.098\n",
      "05/11/2020 16:27:28 - INFO - __main__ -     step:41 loss 9.041\n",
      "05/11/2020 16:27:29 - INFO - __main__ -     step:42 loss 8.197\n",
      "05/11/2020 16:27:30 - INFO - __main__ -     step:43 loss 9.464\n",
      "05/11/2020 16:27:31 - INFO - __main__ -     step:44 loss 8.555\n",
      "05/11/2020 16:27:32 - INFO - __main__ -     step:45 loss 8.008\n",
      "05/11/2020 16:27:33 - INFO - __main__ -     step:46 loss 6.531\n",
      "05/11/2020 16:27:34 - INFO - __main__ -     step:47 loss 8.005\n",
      "05/11/2020 16:27:35 - INFO - __main__ -     step:48 loss 9.709\n",
      "05/11/2020 16:27:36 - INFO - __main__ -     step:49 loss 8.683\n",
      "05/11/2020 16:27:37 - INFO - __main__ -     step:50 loss 10.197\n",
      "05/11/2020 16:27:38 - INFO - __main__ -     step:51 loss 8.573\n",
      "05/11/2020 16:27:39 - INFO - __main__ -     step:52 loss 7.877\n",
      "05/11/2020 16:27:40 - INFO - __main__ -     step:53 loss 9.392\n",
      "05/11/2020 16:27:41 - INFO - __main__ -     step:54 loss 8.569\n",
      "05/11/2020 16:27:42 - INFO - __main__ -     step:55 loss 8.521\n",
      "05/11/2020 16:27:43 - INFO - __main__ -     step:56 loss 8.336\n",
      "05/11/2020 16:27:44 - INFO - __main__ -     step:57 loss 8.712\n",
      "05/11/2020 16:27:45 - INFO - __main__ -     step:58 loss 8.816\n",
      "05/11/2020 16:27:46 - INFO - __main__ -     step:59 loss 7.423\n",
      "05/11/2020 16:27:47 - INFO - __main__ -     step:60 loss 9.684\n",
      "05/11/2020 16:27:49 - INFO - __main__ -     step:61 loss 6.776\n",
      "05/11/2020 16:27:50 - INFO - __main__ -     step:62 loss 9.321\n",
      "05/11/2020 16:27:51 - INFO - __main__ -     step:63 loss 7.489\n",
      "05/11/2020 16:27:52 - INFO - __main__ -     step:64 loss 11.085\n",
      "05/11/2020 16:27:53 - INFO - __main__ -     step:65 loss 8.629\n",
      "05/11/2020 16:27:54 - INFO - __main__ -     step:66 loss 8.076\n",
      "05/11/2020 16:27:55 - INFO - __main__ -     step:67 loss 8.527\n",
      "05/11/2020 16:27:56 - INFO - __main__ -     step:68 loss 7.661\n",
      "05/11/2020 16:27:57 - INFO - __main__ -     step:69 loss 9.543\n",
      "05/11/2020 16:27:58 - INFO - __main__ -     step:70 loss 8.801\n",
      "05/11/2020 16:27:59 - INFO - __main__ -     step:71 loss 8.129\n",
      "05/11/2020 16:28:00 - INFO - __main__ -     step:72 loss 8.024\n",
      "05/11/2020 16:28:01 - INFO - __main__ -     step:73 loss 7.699\n",
      "05/11/2020 16:28:02 - INFO - __main__ -     step:74 loss 7.256\n",
      "05/11/2020 16:28:03 - INFO - __main__ -     step:75 loss 7.344\n",
      "05/11/2020 16:28:04 - INFO - __main__ -     step:76 loss 8.541\n",
      "05/11/2020 16:28:05 - INFO - __main__ -     step:77 loss 8.935\n",
      "05/11/2020 16:28:06 - INFO - __main__ -     step:78 loss 7.799\n",
      "05/11/2020 16:28:07 - INFO - __main__ -     step:79 loss 7.548\n",
      "05/11/2020 16:28:08 - INFO - __main__ -     step:80 loss 7.050\n",
      "05/11/2020 16:28:09 - INFO - __main__ -     step:81 loss 9.297\n",
      "05/11/2020 16:28:10 - INFO - __main__ -     step:82 loss 8.222\n",
      "05/11/2020 16:28:11 - INFO - __main__ -     step:83 loss 8.889\n",
      "05/11/2020 16:28:12 - INFO - __main__ -     step:84 loss 8.409\n",
      "05/11/2020 16:28:13 - INFO - __main__ -     step:85 loss 8.879\n",
      "05/11/2020 16:28:14 - INFO - __main__ -     step:86 loss 8.816\n",
      "05/11/2020 16:28:15 - INFO - __main__ -     step:87 loss 8.724\n",
      "05/11/2020 16:28:16 - INFO - __main__ -     step:88 loss 8.622\n",
      "05/11/2020 16:28:17 - INFO - __main__ -     step:89 loss 9.236\n",
      "05/11/2020 16:28:18 - INFO - __main__ -     step:90 loss 8.143\n",
      "05/11/2020 16:28:19 - INFO - __main__ -     step:91 loss 8.378\n",
      "05/11/2020 16:28:20 - INFO - __main__ -     step:92 loss 7.879\n",
      "05/11/2020 16:28:21 - INFO - __main__ -     step:93 loss 9.474\n",
      "05/11/2020 16:28:23 - INFO - __main__ -     step:94 loss 10.299\n",
      "05/11/2020 16:28:24 - INFO - __main__ -     step:95 loss 8.123\n",
      "05/11/2020 16:28:25 - INFO - __main__ -     step:96 loss 7.704\n",
      "05/11/2020 16:28:26 - INFO - __main__ -     step:97 loss 9.398\n",
      "05/11/2020 16:28:27 - INFO - __main__ -     step:98 loss 8.055\n",
      "05/11/2020 16:28:28 - INFO - __main__ -     step:99 loss 8.424\n",
      "05/11/2020 16:28:29 - INFO - __main__ -     step:100 loss 8.031\n",
      "05/11/2020 16:28:30 - INFO - __main__ -     step:101 loss 7.090\n",
      "05/11/2020 16:28:31 - INFO - __main__ -     step:102 loss 8.352\n",
      "05/11/2020 16:28:32 - INFO - __main__ -     step:103 loss 8.061\n",
      "05/11/2020 16:28:33 - INFO - __main__ -     step:104 loss 8.227\n",
      "05/11/2020 16:28:34 - INFO - __main__ -     step:105 loss 8.786\n",
      "05/11/2020 16:28:35 - INFO - __main__ -     step:106 loss 8.964\n",
      "05/11/2020 16:28:36 - INFO - __main__ -     step:107 loss 9.385\n",
      "05/11/2020 16:28:37 - INFO - __main__ -     step:108 loss 8.128\n",
      "05/11/2020 16:28:38 - INFO - __main__ -     step:109 loss 9.777\n",
      "05/11/2020 16:28:39 - INFO - __main__ -     step:110 loss 7.407\n",
      "05/11/2020 16:28:40 - INFO - __main__ -     step:111 loss 9.012\n",
      "05/11/2020 16:28:41 - INFO - __main__ -     step:112 loss 8.025\n",
      "05/11/2020 16:28:42 - INFO - __main__ -     step:113 loss 8.848\n",
      "05/11/2020 16:28:43 - INFO - __main__ -     step:114 loss 8.230\n",
      "05/11/2020 16:28:44 - INFO - __main__ -     step:115 loss 8.031\n",
      "05/11/2020 16:28:45 - INFO - __main__ -     step:116 loss 7.779\n",
      "05/11/2020 16:28:46 - INFO - __main__ -     step:117 loss 9.404\n",
      "05/11/2020 16:28:47 - INFO - __main__ -     step:118 loss 8.488\n",
      "05/11/2020 16:28:48 - INFO - __main__ -     step:119 loss 7.829\n",
      "05/11/2020 16:28:49 - INFO - __main__ -     step:120 loss 6.982\n",
      "05/11/2020 16:28:50 - INFO - __main__ -     step:121 loss 8.900\n",
      "05/11/2020 16:28:51 - INFO - __main__ -     step:122 loss 8.910\n",
      "05/11/2020 16:28:52 - INFO - __main__ -     step:123 loss 7.736\n",
      "05/11/2020 16:28:53 - INFO - __main__ -     step:124 loss 6.924\n",
      "05/11/2020 16:28:54 - INFO - __main__ -     step:125 loss 9.200\n",
      "05/11/2020 16:28:55 - INFO - __main__ -     step:126 loss 8.459\n",
      "05/11/2020 16:28:56 - INFO - __main__ -     step:127 loss 11.135\n",
      "05/11/2020 16:28:58 - INFO - __main__ -     step:128 loss 9.442\n",
      "05/11/2020 16:28:59 - INFO - __main__ -     step:129 loss 7.579\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/11/2020 16:29:00 - INFO - __main__ -     step:130 loss 8.103\n",
      "05/11/2020 16:29:01 - INFO - __main__ -     step:131 loss 8.775\n",
      "05/11/2020 16:29:02 - INFO - __main__ -     step:132 loss 7.935\n",
      "05/11/2020 16:29:03 - INFO - __main__ -     step:133 loss 8.179\n",
      "05/11/2020 16:29:04 - INFO - __main__ -     step:134 loss 9.230\n",
      "05/11/2020 16:29:05 - INFO - __main__ -     step:135 loss 7.149\n",
      "05/11/2020 16:29:06 - INFO - __main__ -     step:136 loss 8.449\n",
      "05/11/2020 16:29:07 - INFO - __main__ -     step:137 loss 8.932\n",
      "05/11/2020 16:29:08 - INFO - __main__ -     step:138 loss 8.099\n",
      "05/11/2020 16:29:09 - INFO - __main__ -     step:139 loss 9.433\n",
      "05/11/2020 16:29:10 - INFO - __main__ -     step:140 loss 7.274\n",
      "05/11/2020 16:29:11 - INFO - __main__ -     step:141 loss 7.774\n",
      "05/11/2020 16:29:12 - INFO - __main__ -     step:142 loss 7.644\n",
      "05/11/2020 16:29:13 - INFO - __main__ -     step:143 loss 9.361\n",
      "05/11/2020 16:29:14 - INFO - __main__ -     step:144 loss 8.997\n",
      "05/11/2020 16:29:15 - INFO - __main__ -     step:145 loss 7.475\n",
      "05/11/2020 16:29:16 - INFO - __main__ -     step:146 loss 7.185\n",
      "05/11/2020 16:29:17 - INFO - __main__ -     step:147 loss 7.735\n",
      "05/11/2020 16:29:18 - INFO - __main__ -     step:148 loss 8.025\n",
      "05/11/2020 16:29:19 - INFO - __main__ -     step:149 loss 8.075\n",
      "05/11/2020 16:29:20 - INFO - __main__ -     step:150 loss 8.015\n",
      "05/11/2020 16:29:21 - INFO - __main__ -     step:151 loss 8.678\n",
      "05/11/2020 16:29:22 - INFO - __main__ -     step:152 loss 7.296\n",
      "05/11/2020 16:29:23 - INFO - __main__ -     step:153 loss 8.179\n",
      "05/11/2020 16:29:24 - INFO - __main__ -     step:154 loss 8.296\n",
      "05/11/2020 16:29:25 - INFO - __main__ -     step:155 loss 6.482\n",
      "05/11/2020 16:29:26 - INFO - __main__ -     step:156 loss 8.185\n",
      "05/11/2020 16:29:27 - INFO - __main__ -     step:157 loss 7.699\n",
      "05/11/2020 16:29:28 - INFO - __main__ -     step:158 loss 7.947\n",
      "05/11/2020 16:29:29 - INFO - __main__ -     step:159 loss 8.054\n",
      "05/11/2020 16:29:30 - INFO - __main__ -     step:160 loss 6.772\n",
      "05/11/2020 16:29:31 - INFO - __main__ -     step:161 loss 7.545\n",
      "05/11/2020 16:29:32 - INFO - __main__ -     step:162 loss 10.000\n",
      "05/11/2020 16:29:33 - INFO - __main__ -     step:163 loss 6.400\n",
      "05/11/2020 16:29:34 - INFO - __main__ -     step:164 loss 9.085\n",
      "05/11/2020 16:29:35 - INFO - __main__ -     step:165 loss 8.483\n",
      "05/11/2020 16:29:36 - INFO - __main__ -     step:166 loss 7.393\n",
      "05/11/2020 16:29:37 - INFO - __main__ -     step:167 loss 8.400\n",
      "05/11/2020 16:29:39 - INFO - __main__ -     step:168 loss 8.013\n",
      "05/11/2020 16:29:40 - INFO - __main__ -     step:169 loss 9.525\n",
      "05/11/2020 16:29:41 - INFO - __main__ -     step:170 loss 12.706\n",
      "05/11/2020 16:29:42 - INFO - __main__ -     step:171 loss 7.550\n",
      "05/11/2020 16:29:43 - INFO - __main__ -     step:172 loss 9.190\n",
      "05/11/2020 16:29:44 - INFO - __main__ -     step:173 loss 7.448\n",
      "05/11/2020 16:29:45 - INFO - __main__ -     step:174 loss 8.821\n",
      "05/11/2020 16:29:46 - INFO - __main__ -     step:175 loss 8.267\n",
      "05/11/2020 16:29:47 - INFO - __main__ -     step:176 loss 7.659\n",
      "05/11/2020 16:29:48 - INFO - __main__ -     step:177 loss 9.135\n",
      "05/11/2020 16:29:49 - INFO - __main__ -     step:178 loss 7.817\n",
      "05/11/2020 16:29:50 - INFO - __main__ -     step:179 loss 8.182\n",
      "05/11/2020 16:29:51 - INFO - __main__ -     step:180 loss 9.113\n",
      "05/11/2020 16:29:52 - INFO - __main__ -     step:181 loss 7.872\n",
      "05/11/2020 16:29:53 - INFO - __main__ -     step:182 loss 7.185\n",
      "05/11/2020 16:29:54 - INFO - __main__ -     step:183 loss 9.209\n",
      "05/11/2020 16:29:55 - INFO - __main__ -     step:184 loss 7.289\n",
      "05/11/2020 16:29:56 - INFO - __main__ -     step:185 loss 8.438\n",
      "05/11/2020 16:29:57 - INFO - __main__ -     step:186 loss 8.783\n",
      "05/11/2020 16:29:58 - INFO - __main__ -     step:187 loss 8.110\n",
      "05/11/2020 16:29:59 - INFO - __main__ -     step:188 loss 8.976\n",
      "05/11/2020 16:30:00 - INFO - __main__ -     step:189 loss 8.982\n",
      "05/11/2020 16:30:01 - INFO - __main__ -     step:190 loss 8.068\n",
      "05/11/2020 16:30:02 - INFO - __main__ -     step:191 loss 8.736\n",
      "05/11/2020 16:30:03 - INFO - __main__ -     step:192 loss 7.536\n",
      "05/11/2020 16:30:04 - INFO - __main__ -     step:193 loss 7.402\n",
      "05/11/2020 16:30:05 - INFO - __main__ -     step:194 loss 7.680\n",
      "05/11/2020 16:30:06 - INFO - __main__ -     step:195 loss 9.566\n",
      "05/11/2020 16:30:07 - INFO - __main__ -     step:196 loss 8.040\n",
      "05/11/2020 16:30:08 - INFO - __main__ -     step:197 loss 8.177\n",
      "05/11/2020 16:30:09 - INFO - __main__ -     step:198 loss 7.536\n",
      "05/11/2020 16:30:10 - INFO - __main__ -     step:199 loss 9.940\n",
      "05/11/2020 16:30:11 - INFO - __main__ -     step:200 loss 8.194\n",
      "05/11/2020 16:30:12 - INFO - __main__ -     step:201 loss 8.831\n",
      "05/11/2020 16:30:13 - INFO - __main__ -     step:202 loss 9.520\n",
      "05/11/2020 16:30:14 - INFO - __main__ -     step:203 loss 7.732\n",
      "05/11/2020 16:30:15 - INFO - __main__ -     step:204 loss 7.458\n",
      "05/11/2020 16:30:16 - INFO - __main__ -     step:205 loss 8.239\n",
      "05/11/2020 16:30:17 - INFO - __main__ -     step:206 loss 9.558\n",
      "05/11/2020 16:30:18 - INFO - __main__ -     step:207 loss 7.146\n",
      "05/11/2020 16:30:19 - INFO - __main__ -     step:208 loss 8.587\n",
      "05/11/2020 16:30:20 - INFO - __main__ -     step:209 loss 6.023\n",
      "05/11/2020 16:30:22 - INFO - __main__ -     step:210 loss 8.458\n",
      "05/11/2020 16:30:23 - INFO - __main__ -     step:211 loss 7.481\n",
      "05/11/2020 16:30:24 - INFO - __main__ -     step:212 loss 7.820\n",
      "05/11/2020 16:30:25 - INFO - __main__ -     step:213 loss 8.296\n",
      "05/11/2020 16:30:26 - INFO - __main__ -     step:214 loss 8.676\n",
      "05/11/2020 16:30:27 - INFO - __main__ -     step:215 loss 8.812\n",
      "05/11/2020 16:30:28 - INFO - __main__ -     step:216 loss 8.897\n",
      "05/11/2020 16:30:29 - INFO - __main__ -     step:217 loss 7.172\n",
      "05/11/2020 16:30:30 - INFO - __main__ -     step:218 loss 8.493\n",
      "05/11/2020 16:30:31 - INFO - __main__ -     step:219 loss 7.573\n",
      "05/11/2020 16:30:32 - INFO - __main__ -     step:220 loss 7.915\n",
      "05/11/2020 16:30:33 - INFO - __main__ -     step:221 loss 8.141\n",
      "05/11/2020 16:30:34 - INFO - __main__ -     step:222 loss 7.877\n",
      "05/11/2020 16:30:35 - INFO - __main__ -     step:223 loss 8.869\n",
      "05/11/2020 16:30:36 - INFO - __main__ -     step:224 loss 8.703\n",
      "05/11/2020 16:30:37 - INFO - __main__ -     step:225 loss 6.014\n",
      "05/11/2020 16:30:38 - INFO - __main__ -     step:226 loss 9.055\n",
      "05/11/2020 16:30:39 - INFO - __main__ -     step:227 loss 7.908\n",
      "05/11/2020 16:30:40 - INFO - __main__ -     step:228 loss 7.986\n",
      "05/11/2020 16:30:41 - INFO - __main__ -     step:229 loss 8.169\n",
      "05/11/2020 16:30:42 - INFO - __main__ -     step:230 loss 7.657\n",
      "05/11/2020 16:30:43 - INFO - __main__ -     step:231 loss 7.804\n",
      "05/11/2020 16:30:44 - INFO - __main__ -     step:232 loss 8.193\n",
      "05/11/2020 16:30:45 - INFO - __main__ -     step:233 loss 7.248\n",
      "05/11/2020 16:30:46 - INFO - __main__ -     step:234 loss 8.115\n",
      "05/11/2020 16:30:47 - INFO - __main__ -     step:235 loss 8.339\n",
      "05/11/2020 16:30:48 - INFO - __main__ -     step:236 loss 7.358\n",
      "05/11/2020 16:30:49 - INFO - __main__ -     step:237 loss 7.204\n",
      "05/11/2020 16:30:50 - INFO - __main__ -     step:238 loss 8.321\n",
      "05/11/2020 16:30:51 - INFO - __main__ -     step:239 loss 7.557\n",
      "05/11/2020 16:30:52 - INFO - __main__ -     step:240 loss 6.232\n",
      "05/11/2020 16:30:53 - INFO - __main__ -     step:241 loss 11.687\n",
      "05/11/2020 16:30:54 - INFO - __main__ -     step:242 loss 9.197\n",
      "05/11/2020 16:30:55 - INFO - __main__ -     step:243 loss 7.238\n",
      "05/11/2020 16:30:56 - INFO - __main__ -     step:244 loss 8.269\n",
      "05/11/2020 16:30:57 - INFO - __main__ -     step:245 loss 6.956\n",
      "05/11/2020 16:30:58 - INFO - __main__ -     step:246 loss 8.219\n",
      "05/11/2020 16:30:59 - INFO - __main__ -     step:247 loss 8.060\n",
      "05/11/2020 16:31:00 - INFO - __main__ -     step:248 loss 9.203\n",
      "05/11/2020 16:31:01 - INFO - __main__ -     step:249 loss 7.274\n",
      "05/11/2020 16:31:02 - INFO - __main__ -     step:250 loss 8.068\n",
      "05/11/2020 16:31:03 - INFO - __main__ -     step:251 loss 7.699\n",
      "05/11/2020 16:31:05 - INFO - __main__ -     step:252 loss 6.591\n",
      "05/11/2020 16:31:06 - INFO - __main__ -     step:253 loss 9.096\n",
      "05/11/2020 16:31:07 - INFO - __main__ -     step:254 loss 9.516\n",
      "05/11/2020 16:31:08 - INFO - __main__ -     step:255 loss 6.665\n",
      "05/11/2020 16:31:09 - INFO - __main__ -     step:256 loss 7.307\n",
      "05/11/2020 16:31:10 - INFO - __main__ -     step:257 loss 8.097\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/11/2020 16:31:11 - INFO - __main__ -     step:258 loss 7.766\n",
      "05/11/2020 16:31:12 - INFO - __main__ -     step:259 loss 10.236\n",
      "05/11/2020 16:31:13 - INFO - __main__ -     step:260 loss 7.421\n",
      "05/11/2020 16:31:14 - INFO - __main__ -     step:261 loss 8.146\n",
      "05/11/2020 16:31:15 - INFO - __main__ -     step:262 loss 7.902\n",
      "05/11/2020 16:31:16 - INFO - __main__ -     step:263 loss 7.716\n",
      "05/11/2020 16:31:17 - INFO - __main__ -     step:264 loss 8.445\n",
      "05/11/2020 16:31:18 - INFO - __main__ -     step:265 loss 8.077\n",
      "05/11/2020 16:31:19 - INFO - __main__ -     step:266 loss 7.216\n",
      "05/11/2020 16:31:20 - INFO - __main__ -     step:267 loss 8.514\n",
      "05/11/2020 16:31:21 - INFO - __main__ -     step:268 loss 8.970\n",
      "05/11/2020 16:31:22 - INFO - __main__ -     step:269 loss 9.938\n",
      "05/11/2020 16:31:23 - INFO - __main__ -     step:270 loss 7.672\n",
      "05/11/2020 16:31:24 - INFO - __main__ -     step:271 loss 7.879\n",
      "05/11/2020 16:31:25 - INFO - __main__ -     step:272 loss 8.970\n",
      "05/11/2020 16:31:26 - INFO - __main__ -     step:273 loss 7.910\n",
      "05/11/2020 16:31:27 - INFO - __main__ -     step:274 loss 7.371\n",
      "05/11/2020 16:31:28 - INFO - __main__ -     step:275 loss 8.983\n",
      "05/11/2020 16:31:29 - INFO - __main__ -     step:276 loss 7.610\n",
      "05/11/2020 16:31:30 - INFO - __main__ -     step:277 loss 6.349\n",
      "05/11/2020 16:31:31 - INFO - __main__ -     step:278 loss 7.176\n",
      "05/11/2020 16:31:32 - INFO - __main__ -     step:279 loss 7.599\n",
      "05/11/2020 16:31:33 - INFO - __main__ -     step:280 loss 7.190\n",
      "05/11/2020 16:31:34 - INFO - __main__ -     step:281 loss 10.307\n",
      "05/11/2020 16:31:35 - INFO - __main__ -     step:282 loss 6.995\n",
      "05/11/2020 16:31:36 - INFO - __main__ -     step:283 loss 8.073\n",
      "05/11/2020 16:31:37 - INFO - __main__ -     step:284 loss 7.042\n",
      "05/11/2020 16:31:38 - INFO - __main__ -     step:285 loss 8.269\n",
      "05/11/2020 16:31:39 - INFO - __main__ -     step:286 loss 7.115\n",
      "05/11/2020 16:31:40 - INFO - __main__ -     step:287 loss 9.347\n",
      "05/11/2020 16:31:41 - INFO - __main__ -     step:288 loss 7.505\n",
      "05/11/2020 16:31:42 - INFO - __main__ -     step:289 loss 8.046\n",
      "05/11/2020 16:31:43 - INFO - __main__ -     step:290 loss 8.342\n",
      "05/11/2020 16:31:44 - INFO - __main__ -     step:291 loss 7.161\n",
      "05/11/2020 16:31:45 - INFO - __main__ -     step:292 loss 7.910\n",
      "05/11/2020 16:31:46 - INFO - __main__ -     step:293 loss 7.990\n",
      "05/11/2020 16:31:48 - INFO - __main__ -     step:294 loss 6.829\n",
      "05/11/2020 16:31:49 - INFO - __main__ -     step:295 loss 6.837\n",
      "05/11/2020 16:31:50 - INFO - __main__ -     step:296 loss 8.025\n",
      "05/11/2020 16:31:51 - INFO - __main__ -     step:297 loss 8.043\n",
      "05/11/2020 16:31:52 - INFO - __main__ -     step:298 loss 8.812\n",
      "05/11/2020 16:31:53 - INFO - __main__ -     step:299 loss 6.935\n",
      "05/11/2020 16:31:54 - INFO - __main__ -     step:300 loss 7.220\n",
      "05/11/2020 16:31:55 - INFO - __main__ -     step:301 loss 8.390\n",
      "05/11/2020 16:31:56 - INFO - __main__ -     step:302 loss 6.945\n",
      "05/11/2020 16:31:57 - INFO - __main__ -     step:303 loss 8.067\n",
      "05/11/2020 16:31:58 - INFO - __main__ -     step:304 loss 8.738\n",
      "05/11/2020 16:31:59 - INFO - __main__ -     step:305 loss 8.920\n",
      "05/11/2020 16:32:00 - INFO - __main__ -     step:306 loss 8.575\n",
      "05/11/2020 16:32:01 - INFO - __main__ -     step:307 loss 7.955\n",
      "05/11/2020 16:32:02 - INFO - __main__ -     step:308 loss 7.640\n",
      "05/11/2020 16:32:03 - INFO - __main__ -     step:309 loss 6.878\n",
      "05/11/2020 16:32:04 - INFO - __main__ -     step:310 loss 6.571\n",
      "05/11/2020 16:32:06 - INFO - __main__ -     step:311 loss 6.765\n",
      "05/11/2020 16:32:06 - INFO - __main__ -     step:312 loss 12.138\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a43b619e409486a862f413b6da9471a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration', max=313, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/11/2020 16:32:07 - INFO - __main__ -     step:0 loss 7.468\n",
      "05/11/2020 16:32:08 - INFO - __main__ -     step:1 loss 8.239\n",
      "05/11/2020 16:32:09 - INFO - __main__ -     step:2 loss 7.958\n",
      "05/11/2020 16:32:10 - INFO - __main__ -     step:3 loss 7.919\n",
      "05/11/2020 16:32:12 - INFO - __main__ -     step:4 loss 7.376\n",
      "05/11/2020 16:32:13 - INFO - __main__ -     step:5 loss 6.757\n",
      "05/11/2020 16:32:14 - INFO - __main__ -     step:6 loss 7.488\n",
      "05/11/2020 16:32:15 - INFO - __main__ -     step:7 loss 8.289\n",
      "05/11/2020 16:32:16 - INFO - __main__ -     step:8 loss 7.863\n",
      "05/11/2020 16:32:17 - INFO - __main__ -     step:9 loss 6.440\n",
      "05/11/2020 16:32:18 - INFO - __main__ -     step:10 loss 8.580\n",
      "05/11/2020 16:32:19 - INFO - __main__ -     step:11 loss 7.241\n",
      "05/11/2020 16:32:20 - INFO - __main__ -     step:12 loss 7.096\n",
      "05/11/2020 16:32:21 - INFO - __main__ -     step:13 loss 7.500\n",
      "05/11/2020 16:32:22 - INFO - __main__ -     step:14 loss 8.270\n",
      "05/11/2020 16:32:23 - INFO - __main__ -     step:15 loss 6.448\n",
      "05/11/2020 16:32:24 - INFO - __main__ -     step:16 loss 6.542\n",
      "05/11/2020 16:32:25 - INFO - __main__ -     step:17 loss 6.893\n",
      "05/11/2020 16:32:26 - INFO - __main__ -     step:18 loss 7.040\n",
      "05/11/2020 16:32:27 - INFO - __main__ -     step:19 loss 7.239\n",
      "05/11/2020 16:32:28 - INFO - __main__ -     step:20 loss 9.311\n",
      "05/11/2020 16:32:29 - INFO - __main__ -     step:21 loss 6.576\n",
      "05/11/2020 16:32:31 - INFO - __main__ -     step:22 loss 7.198\n",
      "05/11/2020 16:32:32 - INFO - __main__ -     step:23 loss 7.132\n",
      "05/11/2020 16:32:33 - INFO - __main__ -     step:24 loss 7.314\n",
      "05/11/2020 16:32:34 - INFO - __main__ -     step:25 loss 8.412\n",
      "05/11/2020 16:32:35 - INFO - __main__ -     step:26 loss 6.855\n",
      "05/11/2020 16:32:36 - INFO - __main__ -     step:27 loss 6.607\n",
      "05/11/2020 16:32:37 - INFO - __main__ -     step:28 loss 7.913\n",
      "05/11/2020 16:32:38 - INFO - __main__ -     step:29 loss 6.862\n",
      "05/11/2020 16:32:39 - INFO - __main__ -     step:30 loss 6.270\n",
      "05/11/2020 16:32:40 - INFO - __main__ -     step:31 loss 6.371\n",
      "05/11/2020 16:32:41 - INFO - __main__ -     step:32 loss 6.340\n",
      "05/11/2020 16:32:42 - INFO - __main__ -     step:33 loss 8.302\n",
      "05/11/2020 16:32:43 - INFO - __main__ -     step:34 loss 8.092\n",
      "05/11/2020 16:32:44 - INFO - __main__ -     step:35 loss 6.735\n",
      "05/11/2020 16:32:45 - INFO - __main__ -     step:36 loss 7.167\n",
      "05/11/2020 16:32:46 - INFO - __main__ -     step:37 loss 6.186\n",
      "05/11/2020 16:32:47 - INFO - __main__ -     step:38 loss 9.235\n",
      "05/11/2020 16:32:48 - INFO - __main__ -     step:39 loss 8.336\n",
      "05/11/2020 16:32:49 - INFO - __main__ -     step:40 loss 7.233\n",
      "05/11/2020 16:32:50 - INFO - __main__ -     step:41 loss 6.782\n",
      "05/11/2020 16:32:51 - INFO - __main__ -     step:42 loss 7.225\n",
      "05/11/2020 16:32:52 - INFO - __main__ -     step:43 loss 7.721\n",
      "05/11/2020 16:32:53 - INFO - __main__ -     step:44 loss 7.874\n",
      "05/11/2020 16:32:55 - INFO - __main__ -     step:45 loss 6.923\n",
      "05/11/2020 16:32:56 - INFO - __main__ -     step:46 loss 8.386\n",
      "05/11/2020 16:32:57 - INFO - __main__ -     step:47 loss 8.519\n",
      "05/11/2020 16:32:58 - INFO - __main__ -     step:48 loss 6.669\n",
      "05/11/2020 16:32:59 - INFO - __main__ -     step:49 loss 8.429\n",
      "05/11/2020 16:33:00 - INFO - __main__ -     step:50 loss 6.403\n",
      "05/11/2020 16:33:01 - INFO - __main__ -     step:51 loss 6.126\n",
      "05/11/2020 16:33:02 - INFO - __main__ -     step:52 loss 6.591\n",
      "05/11/2020 16:33:03 - INFO - __main__ -     step:53 loss 10.077\n",
      "05/11/2020 16:33:04 - INFO - __main__ -     step:54 loss 10.207\n",
      "05/11/2020 16:33:05 - INFO - __main__ -     step:55 loss 7.760\n",
      "05/11/2020 16:33:06 - INFO - __main__ -     step:56 loss 6.546\n",
      "05/11/2020 16:33:07 - INFO - __main__ -     step:57 loss 6.711\n",
      "05/11/2020 16:33:08 - INFO - __main__ -     step:58 loss 6.880\n",
      "05/11/2020 16:33:09 - INFO - __main__ -     step:59 loss 7.828\n",
      "05/11/2020 16:33:10 - INFO - __main__ -     step:60 loss 7.880\n",
      "05/11/2020 16:33:11 - INFO - __main__ -     step:61 loss 7.548\n",
      "05/11/2020 16:33:13 - INFO - __main__ -     step:62 loss 7.117\n",
      "05/11/2020 16:33:14 - INFO - __main__ -     step:63 loss 7.255\n",
      "05/11/2020 16:33:15 - INFO - __main__ -     step:64 loss 6.813\n",
      "05/11/2020 16:33:16 - INFO - __main__ -     step:65 loss 7.899\n",
      "05/11/2020 16:33:17 - INFO - __main__ -     step:66 loss 7.030\n",
      "05/11/2020 16:33:18 - INFO - __main__ -     step:67 loss 8.524\n",
      "05/11/2020 16:33:19 - INFO - __main__ -     step:68 loss 8.186\n",
      "05/11/2020 16:33:20 - INFO - __main__ -     step:69 loss 7.114\n",
      "05/11/2020 16:33:21 - INFO - __main__ -     step:70 loss 8.138\n",
      "05/11/2020 16:33:22 - INFO - __main__ -     step:71 loss 7.919\n",
      "05/11/2020 16:33:23 - INFO - __main__ -     step:72 loss 7.771\n",
      "05/11/2020 16:33:24 - INFO - __main__ -     step:73 loss 7.111\n",
      "05/11/2020 16:33:25 - INFO - __main__ -     step:74 loss 6.901\n",
      "05/11/2020 16:33:26 - INFO - __main__ -     step:75 loss 6.501\n",
      "05/11/2020 16:33:27 - INFO - __main__ -     step:76 loss 7.868\n",
      "05/11/2020 16:33:28 - INFO - __main__ -     step:77 loss 8.465\n",
      "05/11/2020 16:33:29 - INFO - __main__ -     step:78 loss 9.139\n",
      "05/11/2020 16:33:30 - INFO - __main__ -     step:79 loss 8.620\n",
      "05/11/2020 16:33:31 - INFO - __main__ -     step:80 loss 6.700\n",
      "05/11/2020 16:33:32 - INFO - __main__ -     step:81 loss 7.189\n",
      "05/11/2020 16:33:33 - INFO - __main__ -     step:82 loss 6.921\n",
      "05/11/2020 16:33:34 - INFO - __main__ -     step:83 loss 7.001\n",
      "05/11/2020 16:33:35 - INFO - __main__ -     step:84 loss 6.643\n",
      "05/11/2020 16:33:36 - INFO - __main__ -     step:85 loss 5.852\n",
      "05/11/2020 16:33:37 - INFO - __main__ -     step:86 loss 8.580\n",
      "05/11/2020 16:33:38 - INFO - __main__ -     step:87 loss 7.208\n",
      "05/11/2020 16:33:40 - INFO - __main__ -     step:88 loss 7.782\n",
      "05/11/2020 16:33:41 - INFO - __main__ -     step:89 loss 6.580\n",
      "05/11/2020 16:33:42 - INFO - __main__ -     step:90 loss 7.196\n",
      "05/11/2020 16:33:43 - INFO - __main__ -     step:91 loss 7.216\n",
      "05/11/2020 16:33:44 - INFO - __main__ -     step:92 loss 5.741\n",
      "05/11/2020 16:33:45 - INFO - __main__ -     step:93 loss 7.911\n",
      "05/11/2020 16:33:46 - INFO - __main__ -     step:94 loss 7.343\n",
      "05/11/2020 16:33:47 - INFO - __main__ -     step:95 loss 8.164\n",
      "05/11/2020 16:33:48 - INFO - __main__ -     step:96 loss 6.388\n",
      "05/11/2020 16:33:49 - INFO - __main__ -     step:97 loss 7.443\n",
      "05/11/2020 16:33:50 - INFO - __main__ -     step:98 loss 6.852\n",
      "05/11/2020 16:33:51 - INFO - __main__ -     step:99 loss 6.426\n",
      "05/11/2020 16:33:52 - INFO - __main__ -     step:100 loss 7.901\n",
      "05/11/2020 16:33:53 - INFO - __main__ -     step:101 loss 8.389\n",
      "05/11/2020 16:33:54 - INFO - __main__ -     step:102 loss 6.868\n",
      "05/11/2020 16:33:55 - INFO - __main__ -     step:103 loss 6.335\n",
      "05/11/2020 16:33:56 - INFO - __main__ -     step:104 loss 8.836\n",
      "05/11/2020 16:33:57 - INFO - __main__ -     step:105 loss 6.851\n",
      "05/11/2020 16:33:58 - INFO - __main__ -     step:106 loss 6.719\n",
      "05/11/2020 16:33:59 - INFO - __main__ -     step:107 loss 6.158\n",
      "05/11/2020 16:34:00 - INFO - __main__ -     step:108 loss 8.917\n",
      "05/11/2020 16:34:01 - INFO - __main__ -     step:109 loss 6.596\n",
      "05/11/2020 16:34:02 - INFO - __main__ -     step:110 loss 8.125\n",
      "05/11/2020 16:34:04 - INFO - __main__ -     step:111 loss 7.874\n",
      "05/11/2020 16:34:05 - INFO - __main__ -     step:112 loss 6.894\n",
      "05/11/2020 16:34:06 - INFO - __main__ -     step:113 loss 6.828\n",
      "05/11/2020 16:34:07 - INFO - __main__ -     step:114 loss 8.240\n",
      "05/11/2020 16:34:08 - INFO - __main__ -     step:115 loss 6.672\n",
      "05/11/2020 16:34:09 - INFO - __main__ -     step:116 loss 6.970\n",
      "05/11/2020 16:34:10 - INFO - __main__ -     step:117 loss 6.412\n",
      "05/11/2020 16:34:11 - INFO - __main__ -     step:118 loss 6.812\n",
      "05/11/2020 16:34:12 - INFO - __main__ -     step:119 loss 7.881\n",
      "05/11/2020 16:34:13 - INFO - __main__ -     step:120 loss 5.818\n",
      "05/11/2020 16:34:14 - INFO - __main__ -     step:121 loss 6.953\n",
      "05/11/2020 16:34:15 - INFO - __main__ -     step:122 loss 6.783\n",
      "05/11/2020 16:34:16 - INFO - __main__ -     step:123 loss 6.130\n",
      "05/11/2020 16:34:17 - INFO - __main__ -     step:124 loss 7.082\n",
      "05/11/2020 16:34:18 - INFO - __main__ -     step:125 loss 7.215\n",
      "05/11/2020 16:34:19 - INFO - __main__ -     step:126 loss 6.649\n",
      "05/11/2020 16:34:20 - INFO - __main__ -     step:127 loss 6.500\n",
      "05/11/2020 16:34:21 - INFO - __main__ -     step:128 loss 7.140\n",
      "05/11/2020 16:34:22 - INFO - __main__ -     step:129 loss 6.169\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/11/2020 16:34:23 - INFO - __main__ -     step:130 loss 6.307\n",
      "05/11/2020 16:34:24 - INFO - __main__ -     step:131 loss 8.927\n",
      "05/11/2020 16:34:25 - INFO - __main__ -     step:132 loss 7.149\n",
      "05/11/2020 16:34:26 - INFO - __main__ -     step:133 loss 7.171\n",
      "05/11/2020 16:34:27 - INFO - __main__ -     step:134 loss 7.058\n",
      "05/11/2020 16:34:28 - INFO - __main__ -     step:135 loss 7.312\n",
      "05/11/2020 16:34:29 - INFO - __main__ -     step:136 loss 7.286\n",
      "05/11/2020 16:34:30 - INFO - __main__ -     step:137 loss 7.225\n",
      "05/11/2020 16:34:31 - INFO - __main__ -     step:138 loss 6.819\n",
      "05/11/2020 16:34:32 - INFO - __main__ -     step:139 loss 7.369\n",
      "05/11/2020 16:34:33 - INFO - __main__ -     step:140 loss 6.682\n",
      "05/11/2020 16:34:34 - INFO - __main__ -     step:141 loss 6.892\n",
      "05/11/2020 16:34:35 - INFO - __main__ -     step:142 loss 6.241\n",
      "05/11/2020 16:34:36 - INFO - __main__ -     step:143 loss 6.242\n",
      "05/11/2020 16:34:37 - INFO - __main__ -     step:144 loss 7.169\n",
      "05/11/2020 16:34:38 - INFO - __main__ -     step:145 loss 7.088\n",
      "05/11/2020 16:34:39 - INFO - __main__ -     step:146 loss 5.834\n",
      "05/11/2020 16:34:40 - INFO - __main__ -     step:147 loss 7.876\n",
      "05/11/2020 16:34:41 - INFO - __main__ -     step:148 loss 7.002\n",
      "05/11/2020 16:34:42 - INFO - __main__ -     step:149 loss 9.151\n",
      "05/11/2020 16:34:43 - INFO - __main__ -     step:150 loss 5.935\n",
      "05/11/2020 16:34:45 - INFO - __main__ -     step:151 loss 6.805\n",
      "05/11/2020 16:34:46 - INFO - __main__ -     step:152 loss 7.552\n",
      "05/11/2020 16:34:47 - INFO - __main__ -     step:153 loss 6.492\n",
      "05/11/2020 16:34:48 - INFO - __main__ -     step:154 loss 6.646\n",
      "05/11/2020 16:34:49 - INFO - __main__ -     step:155 loss 8.034\n",
      "05/11/2020 16:34:50 - INFO - __main__ -     step:156 loss 7.266\n",
      "05/11/2020 16:34:51 - INFO - __main__ -     step:157 loss 7.348\n",
      "05/11/2020 16:34:52 - INFO - __main__ -     step:158 loss 7.016\n",
      "05/11/2020 16:34:53 - INFO - __main__ -     step:159 loss 7.454\n",
      "05/11/2020 16:34:54 - INFO - __main__ -     step:160 loss 6.969\n",
      "05/11/2020 16:34:55 - INFO - __main__ -     step:161 loss 7.062\n",
      "05/11/2020 16:34:56 - INFO - __main__ -     step:162 loss 6.142\n",
      "05/11/2020 16:34:57 - INFO - __main__ -     step:163 loss 7.761\n",
      "05/11/2020 16:34:58 - INFO - __main__ -     step:164 loss 5.606\n",
      "05/11/2020 16:34:59 - INFO - __main__ -     step:165 loss 7.267\n",
      "05/11/2020 16:35:00 - INFO - __main__ -     step:166 loss 6.723\n",
      "05/11/2020 16:35:01 - INFO - __main__ -     step:167 loss 5.364\n",
      "05/11/2020 16:35:02 - INFO - __main__ -     step:168 loss 7.961\n",
      "05/11/2020 16:35:03 - INFO - __main__ -     step:169 loss 7.182\n",
      "05/11/2020 16:35:04 - INFO - __main__ -     step:170 loss 6.744\n",
      "05/11/2020 16:35:05 - INFO - __main__ -     step:171 loss 8.830\n",
      "05/11/2020 16:35:06 - INFO - __main__ -     step:172 loss 7.717\n",
      "05/11/2020 16:35:07 - INFO - __main__ -     step:173 loss 6.045\n",
      "05/11/2020 16:35:08 - INFO - __main__ -     step:174 loss 7.558\n",
      "05/11/2020 16:35:09 - INFO - __main__ -     step:175 loss 7.206\n",
      "05/11/2020 16:35:10 - INFO - __main__ -     step:176 loss 7.662\n",
      "05/11/2020 16:35:11 - INFO - __main__ -     step:177 loss 6.588\n",
      "05/11/2020 16:35:12 - INFO - __main__ -     step:178 loss 6.817\n",
      "05/11/2020 16:35:13 - INFO - __main__ -     step:179 loss 7.477\n",
      "05/11/2020 16:35:14 - INFO - __main__ -     step:180 loss 8.094\n",
      "05/11/2020 16:35:15 - INFO - __main__ -     step:181 loss 6.893\n",
      "05/11/2020 16:35:16 - INFO - __main__ -     step:182 loss 7.162\n",
      "05/11/2020 16:35:17 - INFO - __main__ -     step:183 loss 6.314\n",
      "05/11/2020 16:35:18 - INFO - __main__ -     step:184 loss 6.827\n",
      "05/11/2020 16:35:20 - INFO - __main__ -     step:185 loss 6.105\n",
      "05/11/2020 16:35:21 - INFO - __main__ -     step:186 loss 6.155\n",
      "05/11/2020 16:35:22 - INFO - __main__ -     step:187 loss 7.577\n",
      "05/11/2020 16:35:23 - INFO - __main__ -     step:188 loss 6.120\n",
      "05/11/2020 16:35:24 - INFO - __main__ -     step:189 loss 7.233\n",
      "05/11/2020 16:35:25 - INFO - __main__ -     step:190 loss 7.654\n",
      "05/11/2020 16:35:26 - INFO - __main__ -     step:191 loss 7.554\n",
      "05/11/2020 16:35:27 - INFO - __main__ -     step:192 loss 7.116\n",
      "05/11/2020 16:35:28 - INFO - __main__ -     step:193 loss 7.519\n",
      "05/11/2020 16:35:29 - INFO - __main__ -     step:194 loss 6.105\n",
      "05/11/2020 16:35:30 - INFO - __main__ -     step:195 loss 6.398\n",
      "05/11/2020 16:35:31 - INFO - __main__ -     step:196 loss 6.664\n",
      "05/11/2020 16:35:32 - INFO - __main__ -     step:197 loss 7.157\n",
      "05/11/2020 16:35:33 - INFO - __main__ -     step:198 loss 8.162\n",
      "05/11/2020 16:35:34 - INFO - __main__ -     step:199 loss 5.983\n",
      "05/11/2020 16:35:35 - INFO - __main__ -     step:200 loss 7.023\n",
      "05/11/2020 16:35:36 - INFO - __main__ -     step:201 loss 5.963\n",
      "05/11/2020 16:35:37 - INFO - __main__ -     step:202 loss 9.149\n",
      "05/11/2020 16:35:38 - INFO - __main__ -     step:203 loss 8.067\n",
      "05/11/2020 16:35:40 - INFO - __main__ -     step:204 loss 7.084\n",
      "05/11/2020 16:35:41 - INFO - __main__ -     step:205 loss 7.254\n",
      "05/11/2020 16:35:42 - INFO - __main__ -     step:206 loss 7.581\n",
      "05/11/2020 16:35:43 - INFO - __main__ -     step:207 loss 6.502\n",
      "05/11/2020 16:35:44 - INFO - __main__ -     step:208 loss 7.008\n",
      "05/11/2020 16:35:46 - INFO - __main__ -     step:209 loss 6.022\n",
      "05/11/2020 16:35:47 - INFO - __main__ -     step:210 loss 6.923\n",
      "05/11/2020 16:35:48 - INFO - __main__ -     step:211 loss 7.626\n",
      "05/11/2020 16:35:49 - INFO - __main__ -     step:212 loss 8.213\n",
      "05/11/2020 16:35:50 - INFO - __main__ -     step:213 loss 7.291\n",
      "05/11/2020 16:35:51 - INFO - __main__ -     step:214 loss 7.914\n",
      "05/11/2020 16:35:52 - INFO - __main__ -     step:215 loss 7.635\n",
      "05/11/2020 16:35:53 - INFO - __main__ -     step:216 loss 6.292\n",
      "05/11/2020 16:35:54 - INFO - __main__ -     step:217 loss 7.116\n",
      "05/11/2020 16:35:55 - INFO - __main__ -     step:218 loss 6.285\n",
      "05/11/2020 16:35:56 - INFO - __main__ -     step:219 loss 5.640\n",
      "05/11/2020 16:35:57 - INFO - __main__ -     step:220 loss 6.540\n",
      "05/11/2020 16:35:58 - INFO - __main__ -     step:221 loss 5.490\n",
      "05/11/2020 16:35:59 - INFO - __main__ -     step:222 loss 6.558\n",
      "05/11/2020 16:36:00 - INFO - __main__ -     step:223 loss 7.531\n",
      "05/11/2020 16:36:01 - INFO - __main__ -     step:224 loss 7.033\n",
      "05/11/2020 16:36:02 - INFO - __main__ -     step:225 loss 7.634\n",
      "05/11/2020 16:36:03 - INFO - __main__ -     step:226 loss 7.648\n",
      "05/11/2020 16:36:04 - INFO - __main__ -     step:227 loss 6.576\n",
      "05/11/2020 16:36:05 - INFO - __main__ -     step:228 loss 6.064\n",
      "05/11/2020 16:36:06 - INFO - __main__ -     step:229 loss 7.147\n",
      "05/11/2020 16:36:08 - INFO - __main__ -     step:230 loss 9.033\n",
      "05/11/2020 16:36:09 - INFO - __main__ -     step:231 loss 6.629\n",
      "05/11/2020 16:36:10 - INFO - __main__ -     step:232 loss 7.438\n",
      "05/11/2020 16:36:11 - INFO - __main__ -     step:233 loss 6.938\n",
      "05/11/2020 16:36:12 - INFO - __main__ -     step:234 loss 6.883\n",
      "05/11/2020 16:36:13 - INFO - __main__ -     step:235 loss 7.217\n",
      "05/11/2020 16:36:14 - INFO - __main__ -     step:236 loss 6.167\n",
      "05/11/2020 16:36:15 - INFO - __main__ -     step:237 loss 6.026\n",
      "05/11/2020 16:36:16 - INFO - __main__ -     step:238 loss 6.418\n",
      "05/11/2020 16:36:17 - INFO - __main__ -     step:239 loss 6.895\n",
      "05/11/2020 16:36:18 - INFO - __main__ -     step:240 loss 7.016\n",
      "05/11/2020 16:36:19 - INFO - __main__ -     step:241 loss 7.965\n",
      "05/11/2020 16:36:20 - INFO - __main__ -     step:242 loss 7.951\n",
      "05/11/2020 16:36:21 - INFO - __main__ -     step:243 loss 6.940\n",
      "05/11/2020 16:36:22 - INFO - __main__ -     step:244 loss 7.935\n",
      "05/11/2020 16:36:23 - INFO - __main__ -     step:245 loss 6.442\n",
      "05/11/2020 16:36:24 - INFO - __main__ -     step:246 loss 7.704\n",
      "05/11/2020 16:36:25 - INFO - __main__ -     step:247 loss 6.544\n",
      "05/11/2020 16:36:26 - INFO - __main__ -     step:248 loss 6.161\n",
      "05/11/2020 16:36:27 - INFO - __main__ -     step:249 loss 7.664\n",
      "05/11/2020 16:36:28 - INFO - __main__ -     step:250 loss 7.146\n",
      "05/11/2020 16:36:29 - INFO - __main__ -     step:251 loss 5.856\n",
      "05/11/2020 16:36:30 - INFO - __main__ -     step:252 loss 6.636\n",
      "05/11/2020 16:36:31 - INFO - __main__ -     step:253 loss 5.993\n",
      "05/11/2020 16:36:33 - INFO - __main__ -     step:254 loss 6.774\n",
      "05/11/2020 16:36:34 - INFO - __main__ -     step:255 loss 7.385\n",
      "05/11/2020 16:36:35 - INFO - __main__ -     step:256 loss 6.849\n",
      "05/11/2020 16:36:36 - INFO - __main__ -     step:257 loss 6.361\n",
      "05/11/2020 16:36:37 - INFO - __main__ -     step:258 loss 6.476\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/11/2020 16:36:38 - INFO - __main__ -     step:259 loss 7.429\n",
      "05/11/2020 16:36:39 - INFO - __main__ -     step:260 loss 5.865\n",
      "05/11/2020 16:36:40 - INFO - __main__ -     step:261 loss 6.657\n",
      "05/11/2020 16:36:41 - INFO - __main__ -     step:262 loss 5.916\n",
      "05/11/2020 16:36:42 - INFO - __main__ -     step:263 loss 7.265\n",
      "05/11/2020 16:36:43 - INFO - __main__ -     step:264 loss 7.305\n",
      "05/11/2020 16:36:44 - INFO - __main__ -     step:265 loss 7.678\n",
      "05/11/2020 16:36:45 - INFO - __main__ -     step:266 loss 6.456\n",
      "05/11/2020 16:36:46 - INFO - __main__ -     step:267 loss 6.515\n",
      "05/11/2020 16:36:47 - INFO - __main__ -     step:268 loss 9.640\n",
      "05/11/2020 16:36:48 - INFO - __main__ -     step:269 loss 6.633\n",
      "05/11/2020 16:36:49 - INFO - __main__ -     step:270 loss 6.951\n",
      "05/11/2020 16:36:50 - INFO - __main__ -     step:271 loss 7.304\n",
      "05/11/2020 16:36:51 - INFO - __main__ -     step:272 loss 7.222\n",
      "05/11/2020 16:36:52 - INFO - __main__ -     step:273 loss 6.860\n",
      "05/11/2020 16:36:53 - INFO - __main__ -     step:274 loss 6.266\n",
      "05/11/2020 16:36:54 - INFO - __main__ -     step:275 loss 6.050\n",
      "05/11/2020 16:36:55 - INFO - __main__ -     step:276 loss 6.573\n",
      "05/11/2020 16:36:56 - INFO - __main__ -     step:277 loss 6.898\n",
      "05/11/2020 16:36:57 - INFO - __main__ -     step:278 loss 7.771\n",
      "05/11/2020 16:36:58 - INFO - __main__ -     step:279 loss 5.540\n",
      "05/11/2020 16:36:59 - INFO - __main__ -     step:280 loss 7.893\n",
      "05/11/2020 16:37:01 - INFO - __main__ -     step:281 loss 6.831\n",
      "05/11/2020 16:37:02 - INFO - __main__ -     step:282 loss 6.735\n",
      "05/11/2020 16:37:03 - INFO - __main__ -     step:283 loss 5.956\n",
      "05/11/2020 16:37:04 - INFO - __main__ -     step:284 loss 7.367\n",
      "05/11/2020 16:37:05 - INFO - __main__ -     step:285 loss 6.287\n",
      "05/11/2020 16:37:06 - INFO - __main__ -     step:286 loss 7.413\n",
      "05/11/2020 16:37:07 - INFO - __main__ -     step:287 loss 6.661\n",
      "05/11/2020 16:37:08 - INFO - __main__ -     step:288 loss 8.042\n",
      "05/11/2020 16:37:09 - INFO - __main__ -     step:289 loss 6.572\n",
      "05/11/2020 16:37:10 - INFO - __main__ -     step:290 loss 5.331\n",
      "05/11/2020 16:37:11 - INFO - __main__ -     step:291 loss 6.364\n",
      "05/11/2020 16:37:12 - INFO - __main__ -     step:292 loss 6.736\n",
      "05/11/2020 16:37:13 - INFO - __main__ -     step:293 loss 6.530\n",
      "05/11/2020 16:37:14 - INFO - __main__ -     step:294 loss 7.530\n",
      "05/11/2020 16:37:15 - INFO - __main__ -     step:295 loss 6.501\n",
      "05/11/2020 16:37:16 - INFO - __main__ -     step:296 loss 5.994\n",
      "05/11/2020 16:37:17 - INFO - __main__ -     step:297 loss 6.060\n",
      "05/11/2020 16:37:18 - INFO - __main__ -     step:298 loss 6.358\n",
      "05/11/2020 16:37:19 - INFO - __main__ -     step:299 loss 6.426\n",
      "05/11/2020 16:37:20 - INFO - __main__ -     step:300 loss 8.151\n",
      "05/11/2020 16:37:21 - INFO - __main__ -     step:301 loss 7.589\n",
      "05/11/2020 16:37:22 - INFO - __main__ -     step:302 loss 7.229\n",
      "05/11/2020 16:37:23 - INFO - __main__ -     step:303 loss 5.970\n",
      "05/11/2020 16:37:24 - INFO - __main__ -     step:304 loss 6.585\n",
      "05/11/2020 16:37:25 - INFO - __main__ -     step:305 loss 6.160\n",
      "05/11/2020 16:37:26 - INFO - __main__ -     step:306 loss 7.578\n",
      "05/11/2020 16:37:27 - INFO - __main__ -     step:307 loss 5.776\n",
      "05/11/2020 16:37:28 - INFO - __main__ -     step:308 loss 7.235\n",
      "05/11/2020 16:37:29 - INFO - __main__ -     step:309 loss 6.337\n",
      "05/11/2020 16:37:30 - INFO - __main__ -     step:310 loss 6.210\n",
      "05/11/2020 16:37:31 - INFO - __main__ -     step:311 loss 8.335\n",
      "05/11/2020 16:37:32 - INFO - __main__ -     step:312 loss 6.487\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "set_seed(args)\n",
    "model.to(args.device)\n",
    "global_step, tr_loss = train_pointer(args,ner_train_dataset,ner_val_dataset, model, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# BERT+DGCNN+CRF\n",
    "# precison       0.994937     0.866441     0.850437     0.851217     0.857040   \n",
    "# recall         0.994848     0.881341     0.869420     0.824454     0.852126   \n",
    "# f1             0.994893     0.873828     0.859823     0.837622     0.854576"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# BERT+BILSTM+CRF\n",
    "# precison       0.994963     0.866162     0.864721     0.837088     0.853526   \n",
    "# recall         0.994857     0.884781     0.848958     0.843668     0.861682   \n",
    "# f1             0.994910     0.875372     0.856767     0.840365     0.857584   \n",
    "# number    234482.000000  1163.000000  2688.000000  1145.000000  4186.000000 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#p=1\n",
    "# 05/11/2020 00:07:06 - INFO - __main__ -     precision_sample = [0.82914503 0.82265942 0.77135614 0.78173845]\n",
    "# 05/11/2020 00:07:06 - INFO - __main__ -     recall_sample = [0.84112078 0.85131618 0.84109512 0.82025473]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#p=2 thre=0.7\n",
    "# 05/11/2020 10:17:07 - INFO - __main__ -     precision_sample = [0.83593041 0.82177353 0.80729982 0.83111037]\n",
    "# 05/11/2020 10:17:07 - INFO - __main__ -     recall_sample = [0.86575967 0.86575967 0.86849236 0.85310141]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# p=3 thre=0.75\n",
    "# 05/11/2020 10:40:11 - INFO - __main__ -     precision_sample = [0.86567092 0.86749928 0.8531215  0.8389897] \n",
    "# 05/11/2020 10:40:11 - INFO - __main__ -     recall_sample = [0.8870001  0.88445125 0.88584394 0.87956124]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#p=4 thre=0.85\n",
    "# 05/11/2020 11:02:30 - INFO - __main__ -     precision_sample = [0.87749927 0.86942077 0.85802393 0.85333257]\n",
    "# 05/11/2020 11:02:30 - INFO - __main__ -     recall_sample = [0.89464665 0.89379703 0.88858366 0.87591161]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 05/11/2020 16:39:30 - INFO - __main__ -     precision_sample = [0.86551084 0.86393372 0.86565759 0.84778686]\n",
    "# 05/11/2020 16:39:30 - INFO - __main__ -     recall_sample = [0.89124818 0.89549627 0.88858366 0.87408679]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#p=4 thre=0.86 cond seq\n",
    "# 05/11/2020 15:51:29 - INFO - __main__ -     precision_sample = [0.86804907 0.86584291 0.87308652 0.85048298]\n",
    "# 05/11/2020 15:51:29 - INFO - __main__ -     recall_sample = [0.88869933 0.89379703 0.88584394 0.88229847]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#p=5 thre=0.885\n",
    "# 05/11/2020 11:37:38 - INFO - __main__ -     precision_sample = [0.8622072  0.85474792 0.86642521 0.85495418]\n",
    "# 05/11/2020 11:37:38 - INFO - __main__ -     recall_sample = [0.90909014 0.90993975 0.87671153 0.86587512]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rel_labels,rel_preds,token_preds,token_labels=evaluate_pointer(args,ner_val_dataset,model,tokenizer,thre=0.86)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fjw",
   "language": "python",
   "name": "fjw"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
